Imported Upstream version 1.0.1

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.gitignore

@@ -0,0 +1,36 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+*.test
+
+coverage.html
+coverage*.out
+
+*.pyc
+
+xc-out/
+root/
+
+man/aptly.1.html
+man/aptly.1.ronn
+
+.goxc.local.json

.goxc.json

@@ -2,10 +2,12 @@
 	"AppName": "aptly",
 	"ArtifactsDest": "xc-out/",
 	"TasksExclude": [
-		"rmbin"
+		"rmbin",
+		"go-test",
+		"go-vet"
 	],
     "TasksAppend": [
-        "bintray"
+		"bintray"
     ],
     "TaskSettings": {
         "deb": {
@@ -31,8 +33,14 @@
             "downloadspage": "bintray.md"
         }
     },
+	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
 	"Arch": "386 amd64",
 	"Os": "linux darwin freebsd",
-	"MainDirsExclude": "man,_vendor",
+	"MainDirsExclude": "_man,vendor",
+	"BuildSettings": {
+		"LdFlagsXVars": {
+			"Version": "main.Version"
+		}
+	},
 	"ConfigVersion": "0.9"
-}
+}

.travis.yml

@@ -0,0 +1,47 @@
+sudo: false
+
+language: go
+
+go:
+  - 1.6
+  - 1.7
+  - 1.8
+  - tip
+
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+
+env:
+  global:
+    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
+    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+before_install:
+  - virtualenv env
+  - . env/bin/activate
+  - pip install six packaging appdirs
+  - pip install -U pip setuptools
+  - pip install boto requests requests-unixsocket python-swiftclient
+  - mkdir -p $GOPATH/src/github.com/smira
+  - ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/smira || true
+  - cd $GOPATH/src/github.com/smira/aptly
+  - make version
+install:
+  - make prepare
+
+script: make travis
+
+matrix:
+    allow_failures:
+        - go: tip
+
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false

AUTHORS

@@ -0,0 +1,28 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/queeno)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

Makefile

@@ -0,0 +1,84 @@
+GOVERSION=$(shell go version | awk '{print $$3;}')
+VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
+PACKAGES=context database deb files http query swift s3 utils
+PYTHON?=python
+TESTS?=
+BINPATH?=$(GOPATH)/bin
+
+ifeq ($(GOVERSION), devel)
+TRAVIS_TARGET=coveralls
+else
+TRAVIS_TARGET=test
+endif
+
+all: test check system-test
+
+prepare:
+	go get -u github.com/mattn/goveralls
+	go get -u github.com/axw/gocov/gocov
+	go get -u golang.org/x/tools/cmd/cover
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install
+
+dev:
+	go get -u github.com/golang/dep/...
+	go get -u github.com/laher/goxc
+
+coverage.out:
+	rm -f coverage.*.out
+	for i in $(PACKAGES); do go test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
+	echo "mode: count" > coverage.out
+	grep -v -h "mode: count" coverage.*.out >> coverage.out
+	rm -f coverage.*.out
+
+coverage: coverage.out
+	go tool cover -html=coverage.out
+	rm -f coverage.out
+
+check:
+	gometalinter --vendor --vendored-linters --config=linter.json ./...
+
+install:
+	go install -v -ldflags "-X main.Version=$(VERSION)"
+
+system-test: install
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
+
+travis: $(TRAVIS_TARGET) check system-test
+
+test:
+	go test -v `go list ./... | grep -v vendor/` -gocheck.v=true
+
+coveralls: coverage.out
+	$(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+
+mem.png: mem.dat mem.gp
+	gnuplot mem.gp
+	open mem.png
+
+src-package:
+	rm -rf aptly-$(VERSION)
+	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
+	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
+	mkdir -p aptly-$(VERSION)/bash_completion.d
+	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
+	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
+	rm -rf aptly-$(VERSION)
+
+goxc:
+	rm -rf root/
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
+	cp man/aptly.1 root/usr/share/man/man1
+	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	gzip root/usr/share/man/man1/aptly.1
+	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+
+man:
+	make -C man
+
+version:
+	@echo $(VERSION)
+
+.PHONY: coverage.out man version

README.rst

@@ -11,8 +11,8 @@ aptly
 .. image:: https://badges.gitter.im/Join Chat.svg
     :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: http://goreportcard.com/badge/gojp/goreportcard
-    :target: http://goreportcard.com/report/gojp/goreportcard
+.. image:: http://goreportcard.com/badge/smira/aptly
+    :target: http://goreportcard.com/report/smira/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -64,19 +64,14 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.4+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
 
-    go get -u github.com/mattn/gom
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
     cd $GOPATH/src/github.com/smira/aptly
-    gom -production install
-    gom build -o $GOPATH/bin/aptly
+    make install
 
-Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
-should work as well, but might fail or produce different result (if external libraries got updated).
-
-If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+Binary would be installed to ```$GOPATH/bin/aptly``.
 
 Integrations
 ------------
@@ -98,6 +93,8 @@ With configuration management systems:
     (Heavy Water Operations, LLC)
 -   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
     Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
 -   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
     Forrest Alvarez and Brian Jackson
 -   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
@@ -105,6 +102,7 @@ With configuration management systems:
 CLI for aptly API:
 
 -   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
 
 Scala sbt:
 

_man/gen.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/smira/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func allFlags(flags *flag.FlagSet) []*flag.Flag {
+	result := []*flag.Flag{}
+	flags.VisitAll(func(f *flag.Flag) {
+		result = append(result, f)
+	})
+	return result
+}
+
+func findCommand(cmd *commander.Command, name string) (*commander.Command, error) {
+	for _, c := range cmd.Subcommands {
+		if c.Name() == name {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("command %s not found", name)
+}
+
+func capitalize(s string) string {
+	parts := strings.Split(s, " ")
+	for i, part := range parts {
+		if part[0] != '<' && part[0] != '[' && part[len(part)-1] != '>' && part[len(part)-1] != ']' {
+			parts[i] = "`" + part + "`"
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
+func main() {
+	command := cmd.RootCommand()
+	command.UsageLine = "aptly"
+	command.Dispatch(nil)
+
+	_File, _ := filepath.Abs("./man")
+
+	templ := template.New("man").Funcs(template.FuncMap{
+		"allFlags":    allFlags,
+		"findCommand": findCommand,
+		"toUpper":     strings.ToUpper,
+		"capitalize":  capitalize,
+		"authors":     authors,
+	})
+	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
+
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
+	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = templ.ExecuteTemplate(output, "main", command)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	output.Close()
+
+	out, err := exec.Command("ronn", filepath.Join(filepath.Dir(_File), "aptly.1.ronn")).CombinedOutput()
+	if err != nil {
+		os.Stdout.Write(out)
+		log.Fatal(err)
+	}
+
+	cmd := exec.Command("man", filepath.Join(filepath.Dir(_File), "aptly.1"))
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

api/api.go

@@ -0,0 +1,160 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+const (
+	acquiredb = iota
+	releasedb
+)
+
+// Flushes all collections which cache in-memory objects
+func flushColections() {
+	// lock everything to eliminate in-progress calls
+	r := context.CollectionFactory().RemoteRepoCollection()
+	r.Lock()
+	defer r.Unlock()
+
+	l := context.CollectionFactory().LocalRepoCollection()
+	l.Lock()
+	defer l.Unlock()
+
+	s := context.CollectionFactory().SnapshotCollection()
+	s.Lock()
+	defer s.Unlock()
+
+	p := context.CollectionFactory().PublishedRepoCollection()
+	p.Lock()
+	defer p.Unlock()
+
+	// all collections locked, flush them
+	context.CollectionFactory().Flush()
+}
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches. If the two channels are provided,
+// they are used to acquire and release the database.
+//
+// Should be run in goroutine!
+func cacheFlusher(requests chan int, acks chan error) {
+	ticker := time.Tick(15 * time.Minute)
+
+	for {
+		<-ticker
+
+		// if aptly API runs in -no-lock mode,
+		// caches are flushed when DB is closed anyway, no need
+		// to flush them here
+		if requests == nil {
+			flushColections()
+		}
+	}
+}
+
+// Acquire database lock and release it when not needed anymore. Two
+// channels must be provided. The first one is to receive requests to
+// acquire/release the database and the second one is to send acks.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests chan int, acks chan error) {
+	clients := 0
+	for {
+		request := <-requests
+		switch request {
+		case acquiredb:
+			if clients == 0 {
+				acks <- context.ReOpenDatabase()
+			} else {
+				acks <- nil
+			}
+			clients++
+		case releasedb:
+			clients--
+			if clients == 0 {
+				flushColections()
+				acks <- context.CloseDatabase()
+			} else {
+				acks <- nil
+			}
+		}
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			return
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/files.go

@@ -0,0 +1,186 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.Fail(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.Fail(404, err)
+		} else {
+			c.Fail(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.Fail(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+)
+
+// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
+
+	factory := context.CollectionFactory()
+
+	factory.RemoteRepoCollection().RLock()
+	defer factory.RemoteRepoCollection().RUnlock()
+	factory.LocalRepoCollection().RLock()
+	defer factory.LocalRepoCollection().RUnlock()
+	factory.SnapshotCollection().RLock()
+	defer factory.SnapshotCollection().RUnlock()
+	factory.PublishedRepoCollection().RLock()
+	defer factory.PublishedRepoCollection().RUnlock()
+
+	graph, err := deb.BuildGraph(factory, layout)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/publish.go

@@ -0,0 +1,355 @@
+package api
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (utils.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	localCollection := context.CollectionFactory().LocalRepoCollection()
+	localCollection.RLock()
+	defer localCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution   string
+		Label          string
+		Origin         string
+		ForceOverwrite bool
+		SkipContents   *bool
+		Architectures  []string
+		Signing        SigningOptions
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var sources []interface{}
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := context.CollectionFactory().SnapshotCollection()
+		snapshotCollection.RLock()
+		defer snapshotCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == "local" {
+		var localRepo *deb.LocalRepo
+
+		localCollection := context.CollectionFactory().LocalRepoCollection()
+		localCollection.RLock()
+		defer localCollection.RUnlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+	published.Origin = b.Origin
+	published.Label = b.Label
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	duplicate := collection.CheckDuplicate(published)
+	if duplicate != nil {
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		return
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	err = collection.Add(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	c.JSON(201, published)
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipContents   *bool
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+
+	if published.SourceKind == "local" {
+		if len(b.Snapshots) > 0 {
+			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			if err != nil {
+				c.Fail(404, err)
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+		}
+	} else {
+		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	err = collection.Update(published)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	c.JSON(200, published)
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.RLock()
+	defer localRepoCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := collection.Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress(), force)
+	if err != nil {
+		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/repos.go

@@ -2,13 +2,14 @@ package api
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
-	"os"
-	"path/filepath"
 )
 
 // GET /api/repos
@@ -60,9 +61,9 @@ func apiReposCreate(c *gin.Context) {
 // PUT /api/repos/:name
 func apiReposEdit(c *gin.Context) {
 	var b struct {
-		Comment             string
-		DefaultDistribution string
-		DefaultComponent    string
+		Comment             *string
+		DefaultDistribution *string
+		DefaultComponent    *string
 	}
 
 	if !c.Bind(&b) {
@@ -79,14 +80,14 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
-	if b.Comment != "" {
-		repo.Comment = b.Comment
+	if b.Comment != nil {
+		repo.Comment = *b.Comment
 	}
-	if b.DefaultDistribution != "" {
-		repo.DefaultDistribution = b.DefaultDistribution
+	if b.DefaultDistribution != nil {
+		repo.DefaultDistribution = *b.DefaultDistribution
 	}
-	if b.DefaultComponent != "" {
-		repo.DefaultComponent = b.DefaultComponent
+	if b.DefaultComponent != nil {
+		repo.DefaultComponent = *b.DefaultComponent
 	}
 
 	err = collection.Update(repo)

api/router.go

@@ -0,0 +1,113 @@
+package api
+
+import (
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+	ctx "github.com/smira/aptly/context"
+)
+
+var context *ctx.AptlyContext
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	router := gin.Default()
+	router.Use(gin.ErrorLogger())
+
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan int)
+		acks := make(chan error)
+
+		go acquireDatabase(requests, acks)
+		go cacheFlusher(requests, acks)
+
+		router.Use(func(c *gin.Context) {
+			requests <- acquiredb
+			err := <-acks
+			if err != nil {
+				c.Fail(500, err)
+				return
+			}
+			defer func() {
+				requests <- releasedb
+				err = <-acks
+				if err != nil {
+					c.Fail(500, err)
+					return
+				}
+			}()
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher(nil, nil)
+	}
+
+	root := router.Group("/api")
+
+	{
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,411 @@
+package api
+
+import (
+	"fmt"
+
+	"github.com/gin-gonic/gin"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().RemoteRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		c.Fail(409, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.Fail(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.Fail(500, err)
+			return
+		}
+	}
+
+	list := deb.NewPackageList()
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.Fail(500, err)
+			}
+			return
+		}
+		err = list.Add(p)
+		if err != nil {
+			c.Fail(400, err)
+			return
+		}
+	}
+
+	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.Fail(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if !c.Bind(&b) {
+		return
+	}
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	_, err = collection.ByName(b.Name)
+	if err == nil {
+		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		return
+	}
+
+	if b.Name != "" {
+		snapshot.Name = b.Name
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	published := publishedCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.Fail(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.Fail(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList())
+}

aptly/interfaces.go

@@ -3,8 +3,9 @@
 package aptly
 
 import (
-	"github.com/smira/aptly/utils"
 	"io"
+
+	"github.com/smira/aptly/utils"
 )
 
 // PackagePool is asbtraction of package pool storage.
@@ -82,7 +83,7 @@ type Downloader interface {
 	// Download starts new download task
 	Download(url string, destination string, result chan<- error)
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
+	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
 	// Pause pauses task processing
 	Pause()
 	// Resume resumes task processing

aptly/version.go

@@ -0,0 +1,7 @@
+package aptly
+
+// Version of aptly (filled in at link time)
+var Version string
+
+// EnableDebug triggers some debugging features
+const EnableDebug = false

bash_completion.d/aptly

@@ -1,620 +0,0 @@
-#!/bin/bash
-
-# (The MIT License)
-#
-# Copyright (c) 2014 Andrey Smirnov
-#
-# Permission is hereby granted, free of charge, to any person obtaining a copy
-# of this software and associated documentation files (the 'Software'), to deal
-# in the Software without restriction, including without limitation the rights
-# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the Software is
-# furnished to do so, subject to the following conditions:
-#
-# The above copyright notice and this permission notice shall be included in all
-# copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-# SOFTWARE.
-
-__aptly_mirror_list()
-{
-  aptly mirror list -raw
-}
-
-__aptly_repo_list()
-{
-  aptly repo list -raw
-}
-
-__aptly_snapshot_list()
-{
-  aptly snapshot list -raw
-}
-
-__aptly_published_distributions()
-{
-  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
-}
-
-__aptly_published_prefixes()
-{
-  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
-}
-
-__aptly_prefixes_for_distribution()
-{
-  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
-}
-
-_aptly()
-{
-    cur="${COMP_WORDS[COMP_CWORD]}"
-    prev="${COMP_WORDS[COMP_CWORD-1]}"
-
-    commands="api config db graph mirror package publish repo serve snapshot task version"
-    options="-architectures= -config= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests"
-
-    db_subcommands="cleanup recover"
-    mirror_subcommands="create drop show list rename search update"
-    publish_subcommands="drop list repo snapshot switch update"
-    snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
-    repo_subcommands="add copy create drop edit import include list move remove rename search show"
-    package_subcommands="search show"
-    task_subcommands="run"
-    config_subcommands="show"
-    api_subcommands="serve"
-
-    local cmd subcmd numargs numoptions i
-
-    numargs=0
-    numoptions=0
-
-    for (( i=1; i < $COMP_CWORD; i++ )); do
-      if [[ -n "$cmd" ]]; then
-        if [[ ! -n "$subcmd" ]]; then
-          subcmd=${COMP_WORDS[i]}
-          numargs=$(( COMP_CWORD - i - 1 ))
-        else
-          if [[ "${COMP_WORDS[i]}" == -* ]]; then
-            numoptions=$(( numoptions + 1 ))
-            numargs=$(( numargs - 1 ))
-          fi
-        fi
-      else
-        if [[ ! "${COMP_WORDS[i]}" == -* ]]; then
-          cmd=${COMP_WORDS[i]}
-        fi
-      fi
-    done
-
-    if [[ ! -n "$cmd" ]];
-    then
-      case "$cur" in
-            -*)
-              COMPREPLY=($(compgen -W "${options}" -- ${cur}))
-              return 0
-            ;;
-            *)
-              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
-              return 0
-            ;;
-        esac
-    fi
-
-    if [[ ! -n "$subcmd" ]];
-    then
-        case "$prev" in
-            "db")
-              COMPREPLY=($(compgen -W "${db_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "mirror")
-              COMPREPLY=($(compgen -W "${mirror_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "repo")
-              COMPREPLY=($(compgen -W "${repo_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "snapshot")
-              COMPREPLY=($(compgen -W "${snapshot_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "publish")
-              COMPREPLY=($(compgen -W "${publish_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "package")
-              COMPREPLY=($(compgen -W "${package_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "task")
-              COMPREPLY=($(compgen -W "${task_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "config")
-              COMPREPLY=($(compgen -W "${config_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            "api")
-              COMPREPLY=($(compgen -W "${api_subcommands}" -- ${cur}))
-              return 0
-            ;;
-            *)
-            ;;
-        esac
-    fi
-
-    case "$cmd" in
-      "mirror")
-        case "$subcmd" in
-          "create")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
-                return 0
-              fi
-            fi
-          ;;
-          "edit")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "show")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "search")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "rename")
-            if [[ $numargs -eq 0 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "drop")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "list")
-            if [[ $numargs -eq 0 ]]; then
-                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "update")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring=" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "repo")
-        case "$subcmd" in
-          "add")
-            case $numargs in
-              0)
-                if [[ "$cur" == -* ]]; then
-                  COMPREPLY=($(compgen -W "-force-replace -remove-files" -- ${cur}))
-                else
-                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                fi
-                return 0
-              ;;
-              1)
-                local files=$(find . -mindepth 1 -maxdepth 1  \( -type d -or -name \*.deb -or -name \*.dsc -or -name \*.udeb \) -exec basename {} \;)
-                COMPREPLY=($(compgen -W "${files}" -- ${cur}))
-                return 0
-              ;;
-            esac
-          ;;
-          "copy"|"move")
-            case $numargs in
-              0)
-                if [[ "$cur" == -* ]]; then
-                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
-                else
-                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                fi
-                return 0
-              ;;
-              1)
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                return 0
-              ;;
-            esac
-          ;;
-          "create")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
-                return 0
-              fi
-            fi
-          ;;
-          "drop")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "edit")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "search")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "list")
-            if [[ $numargs -eq 0 ]]; then
-                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "include")
-            case $numargs in
-              0)
-                if [[ "$cur" == -* ]]; then
-                  COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
-                else
-                  comptopt -o filenames 2>/dev/null
-                  COMPREPLY=($(compgen -f -- ${cur}))
-                  return 0
-                fi
-                return 0
-              ;;
-            esac
-          ;;
-          "import")
-            case $numargs in
-              0)
-                if [[ "$cur" == -* ]]; then
-                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
-                else
-                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-                fi
-                return 0
-              ;;
-              1)
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                return 0
-              ;;
-            esac
-          ;;
-          "remove")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-dry-run" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "show")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "rename")
-            if [[ $numargs -eq 0 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "snapshot")
-        case "$subcmd" in
-          "create")
-            case $numargs in
-              1)
-                COMPREPLY=($(compgen -W "from empty" -- ${cur}))
-                return 0
-              ;;
-              2)
-                if [[ "$prev" == "from" ]]; then
-                  COMPREPLY=($(compgen -W "mirror repo" -- ${cur}))
-                  return 0
-                fi
-              ;;
-              3)
-                if [[ "$prev" == "mirror" ]]; then
-                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
-                  return 0
-                fi
-                if [[ "$prev" == "repo" ]]; then
-                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                  return 0
-                fi
-              ;;
-            esac
-          ;;
-          "diff")
-            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
-              COMPREPLY=($(compgen -W "-only-matching" -- ${cur}))
-              return 0
-            fi
-
-            if [[ $numargs -lt 2 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "drop")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "list")
-            if [[ $numargs -eq 0 ]]; then
-                COMPREPLY=($(compgen -W "-raw -sort=" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "merge")
-            if [[ $numargs -gt 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-latest" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "pull")
-            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
-              COMPREPLY=($(compgen -W "-all-matches -dry-run -no-deps -no-remove" -- ${cur}))
-              return 0
-            fi
-
-            if [[ $numargs -lt 2 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "filter")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-deps" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "show")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "search")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "rename")
-            if [[ $numargs -eq 0 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "verify")
-            if [[ $numargs -eq 0 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "publish")
-        case "$subcmd" in
-          "snapshot"|"repo")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
-              else
-                if [[ "$subcmd" == "snapshot" ]]; then
-                  COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-                else
-                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
-                fi
-              fi
-              return 0
-            fi
-
-            if [[ $numargs -eq 1 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_published_prefixes)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "list")
-            if [[ $numargs -eq 0 ]]; then
-                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "update")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
-              fi
-              return 0
-            fi
-
-            if [[ $numargs -eq 1 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "switch")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
-              fi
-              return 0
-            fi
-
-            if [[ $numargs -eq 1 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
-              return 0
-            fi
-
-            if [[ $numargs -ge 2 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
-              return 0
-            fi
-          ;;
-          "drop")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force-drop" -- ${cur}))
-              else
-                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
-              fi
-              return 0
-            fi
-
-            if [[ $numargs -eq 1 ]]; then
-              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "package")
-        case "$subcmd" in
-          "search")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-format=" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-          "show")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-files -with-references" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "serve")
-        if [[ "$cur" == -* ]]; then
-          COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
-          return 0
-        fi
-      ;;
-      "graph")
-        if [[ "$cur" == -* ]]; then
-          COMPREPLY=($(compgen -W "-format= -output=" -- ${cur}))
-          return 0
-        fi
-      ;;
-      "api")
-        case "$subcmd" in
-          "serve")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-      "db")
-        case "$subcmd" in
-          "cleanup")
-            if [[ $numargs -eq 0 ]]; then
-              if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur}))
-              fi
-              return 0
-            fi
-          ;;
-        esac
-      ;;
-    esac
-} && complete -F _aptly aptly

cmd/api_serve.go

@@ -0,0 +1,106 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/smira/aptly/api"
+	"github.com/smira/aptly/systemd/activation"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+		os.Remove(file)
+		listener, err := net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -4,13 +4,14 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"text/template"
+	"time"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"text/template"
-	"time"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList

cmd/config_show.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+
 	"github.com/smira/commander"
 )
 

cmd/db_cleanup.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
-	"sort"
-	"strings"
 )
 
 // aptly db cleanup
@@ -153,7 +154,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Substract(existingPackageRefs)
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
 	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())

cmd/graph.go

@@ -0,0 +1,149 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+)
+
+func aptlyGraph(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	layout := context.Flags().Lookup("layout").Value.String()
+
+	fmt.Printf("Generating graph...\n")
+	graph, err := deb.BuildGraph(context.CollectionFactory(), layout)
+
+	if err != nil {
+		return err
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	if err != nil {
+		return err
+	}
+	tempfile.Close()
+	os.Remove(tempfile.Name())
+
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
+
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	err = command.Start()
+	if err != nil {
+		return fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err)
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		return err
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		return err
+	}
+
+	err = command.Wait()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = os.Remove(tempfilename)
+	}()
+
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		if err = viewer.Start(); err == nil {
+			// Wait for a second so that the visualizer has a chance to
+			// open the input file. This needs to be done even if we're
+			// waiting for the visualizer as it can be just a wrapper that
+			// spawns a browser tab and returns right away.
+			defer func(t <-chan time.Time) {
+				<-t
+			}(time.After(time.Second))
+		}
+	}
+
+	return err
+}
+
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
+func makeCmdGraph() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyGraph,
+		UsageLine: "graph",
+		Short:     "render graph of relationships",
+		Long: `
+Command graph displays relationship between mirrors, local repositories,
+snapshots and published repositories using graphviz package to render
+graph as an image.
+
+Example:
+
+  $ aptly graph
+`,
+	}
+
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
+	return cmd
+}

cmd/mirror.go

@@ -1,10 +1,11 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {

cmd/mirror_create.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
@@ -43,6 +44,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	repo.Filter = context.Flags().Lookup("filter").Value.String()
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -97,6 +99,7 @@ Example:
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/mirror_edit.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"

cmd/mirror_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {

cmd/mirror_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/mirror_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdMirrorSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search mirror for packages matching query",
 		Long: `
 Command search displays list of packages in mirror that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/mirror_show.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {

cmd/mirror_update.go

@@ -2,14 +2,15 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"os/signal"
-	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
@@ -40,6 +41,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
@@ -52,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -87,7 +89,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	defer func() {
 		// on any interruption, unlock the mirror
-		err := context.ReOpenDatabase()
+		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
 			context.CollectionFactory().RemoteRepoCollection().Update(repo)
@@ -121,7 +123,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// In separate goroutine (to avoid blocking main), push queue to downloader
 	go func() {
 		for _, task := range queue {
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
 		}
 
 		// We don't need queue after this point
@@ -129,7 +131,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}()
 
 	// Wait for all downloads to finish
-	errors := make([]string, 0)
+	var errors []string
 
 	for count > 0 {
 		select {
@@ -148,7 +150,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	signal.Stop(sigch)
 
 	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
 	err = context.ReOpenDatabase()
@@ -187,6 +189,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package_search.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	if len(args) == 1 {
+		q, err = query.Parse(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format)
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search [<package-query>]",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query.
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/package_show.go

@@ -3,11 +3,12 @@ package cmd
 import (
 	"bufio"
 	"fmt"
+	"os"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
 func printReferencesTo(p *deb.Package) (err error) {
@@ -125,7 +126,7 @@ inclusion into mirrors/snapshots/local repos is shown.
 
 Example:
 
-    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
 `,
 		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
 	}

cmd/publish.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+
+}
+
+func makeCmdPublish() *commander.Command {
+	return &commander.Command{
+		UsageLine: "publish",
+		Short:     "manage published repositories",
+		Subcommands: []*commander.Command{
+			makeCmdPublishDrop(),
+			makeCmdPublishList(),
+			makeCmdPublishRepo(),
+			makeCmdPublishSnapshot(),
+			makeCmdPublishSwitch(),
+			makeCmdPublishUpdate(),
+			makeCmdPublishShow(),
+		},
+	}
+}

cmd/publish_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/publish_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {

cmd/publish_show.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for component, sourceID := range repo.Sources {
+		var name string
+		if repo.SourceKind == "snapshot" {
+			source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			if err != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == "local" {
+			source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			if err != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {

cmd/publish_switch.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {

cmd/publish_update.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"

cmd/repo_add.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {

cmd/repo_create.go

@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoCreate(cmd *commander.Command, args []string) error {
+	var err error
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add local repo: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s successfully added.\nYou can run 'aptly repo add %s ...' to add packages to repository.\n", repo, repo.Name)
+	return err
+}
+
+func makeCmdRepoCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoCreate,
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
+		Short:     "create local repository",
+		Long: `
+Create local package repository. Repository would be empty when
+created, packages could be added from files, copied or moved from
+another local repository or imported from the mirror.
+
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
+Example:
+
+  $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
+
+	return cmd
+}

cmd/repo_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_edit.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/AlekSi/pointer"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"

cmd/repo_include.go

@@ -3,15 +3,16 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
+	"text/template"
+
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"text/template"
 )
 
 func aptlyRepoInclude(cmd *commander.Command, args []string) error {

cmd/repo_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {

cmd/repo_move.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {

cmd/repo_remove.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"

cmd/repo_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/repo_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdRepoSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search repo for packages matching query",
 		Long: `
 Command search displays list of packages in local repository that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'

cmd/repo_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/run.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	ctx "github.com/smira/aptly/context"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*ctx.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			ctx.Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,16 +2,17 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
@@ -22,6 +23,17 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
 	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil

cmd/snapshot_create.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_diff.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_filter.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
-	"strings"
 )
 
 func aptlySnapshotFilter(cmd *commander.Command, args []string) error {

cmd/snapshot_list.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_merge.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"strings"
 )
 
 func aptlySnapshotMerge(cmd *commander.Command, args []string) error {

cmd/snapshot_pull.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+	"strings"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
-	"strings"
 )
 
 func aptlySnapshotPull(cmd *commander.Command, args []string) error {

cmd/snapshot_rename.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 )

cmd/snapshot_search.go

@@ -2,16 +2,21 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 2 {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
@@ -68,9 +73,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	list.PrepareIndex()
 
-	q, err := query.Parse(args[1])
-	if err != nil {
-		return fmt.Errorf("unable to search: %s", err)
+	if len(args) == 2 {
+		q, err = query.Parse(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
 	}
 
 	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
@@ -109,11 +118,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 func makeCmdSnapshotSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search snapshot for packages matching query",
 		Long: `
 Command search displays list of packages in snapshot that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/snapshot_show.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+
+	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", snapshot.Name)
+	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
+	fmt.Printf("Description: %s\n", snapshot.Description)
+	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+	if len(snapshot.SourceIDs) > 0 {
+		fmt.Printf("Sources:\n")
+		for _, sourceID := range snapshot.SourceIDs {
+			var name string
+			if snapshot.SourceKind == "snapshot" {
+				source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == "local" {
+				source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == "repo" {
+				source, err := context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			}
+
+			if name != "" {
+				fmt.Printf("  %s [%s]\n", name, snapshot.SourceKind)
+			}
+		}
+	}
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		ListPackagesRefList(snapshot.RefList())
+	}
+
+	return err
+}
+
+func makeCmdSnapshotShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotShow,
+		UsageLine: "show <name>",
+		Short:     "shows details about snapshot",
+		Long: `
+Command show displays full information about a snapshot.
+
+Example:
+
+    $ aptly snapshot show wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-packages", false, "show list of packages")
+
+	return cmd
+}

cmd/snapshot_verify.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
+
 	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlySnapshotVerify(cmd *commander.Command, args []string) error {

cmd/task_run.go

@@ -20,7 +20,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 		var finfo os.FileInfo
 		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
-			return fmt.Errorf("no such file, %s\n", filename)
+			return fmt.Errorf("no such file, %s", filename)
 		}
 
 		fmt.Print("Reading file...\n\n")

cmd/version.go

@@ -0,0 +1,32 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/commander"
+)
+
+func aptlyVersion(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	fmt.Printf("aptly version: %s\n", aptly.Version)
+	return nil
+}
+
+func makeCmdVersion() *commander.Command {
+	return &commander.Command{
+		Run:       aptlyVersion,
+		UsageLine: "version",
+		Short:     "display version",
+		Long: `
+Shows aptly version.
+
+ex:
+  $ aptly version
+`,
+	}
+}

console/progress.go

@@ -2,10 +2,11 @@ package console
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/cheggaaa/pb"
 	"github.com/smira/aptly/aptly"
 	"github.com/wsxiaoys/terminal/color"
-	"strings"
 )
 
 const (

console/terminal.go

@@ -0,0 +1,12 @@
+package console
+
+import (
+	"syscall"
+
+	"golang.org/x/crypto/ssh/terminal"
+)
+
+// RunningOnTerminal checks whether stdout is terminal
+func RunningOnTerminal() bool {
+	return terminal.IsTerminal(syscall.Stdout)
+}

context/context.go

@@ -0,0 +1,493 @@
+// Package context provides single entry to all resources
+package context
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/s3"
+	"github.com/smira/aptly/swift"
+	"github.com/smira/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	sync.Mutex
+
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.config()
+}
+
+func (context *AptlyContext) config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func (context *AptlyContext) LookupOption(defaultValue bool, name string) (result bool) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.lookupOption(defaultValue, name)
+}
+
+func (context *AptlyContext) lookupOption(defaultValue bool, name string) (result bool) {
+	result = defaultValue
+
+	if context.globalFlags.IsSet(name) {
+		result = context.globalFlags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.lookupOption(context.config().DepFollowSuggests, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.lookupOption(context.config().DepFollowRecommends, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.lookupOption(context.config().DepFollowAllVariants, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.architecturesList == nil {
+		context.architecturesList = context.config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._progress()
+}
+
+func (context *AptlyContext) _progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+			downloadLimit*1024, context._progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.dbPath()
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) dbPath() string {
+	return filepath.Join(context.config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._database()
+}
+
+func (context *AptlyContext) _database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.OpenDB(context.dbPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't open database: %s", err)
+		}
+	}
+
+	return context.database, nil
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.collectionFactory == nil {
+		db, err := context._database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.config().RootDir)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	context.Lock()
+	defer context.Unlock()
+
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(
+				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
+				params.ForceSigV2, params.Debug)
+			if err != nil {
+				Fatal(err)
+			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UploadPath builds path to upload storage
+func (context *AptlyContext) UploadPath() string {
+	return filepath.Join(context.Config().RootDir, "upload")
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.Lock()
+	defer context.Unlock()
+
+	context.flags = flags
+}
+
+// Flags returns current command flags
+func (context *AptlyContext) Flags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.flags
+}
+
+// GlobalFlags returns flags passed to all commands
+func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.globalFlags
+}
+
+// Shutdown shuts context down
+func (context *AptlyContext) Shutdown() {
+	context.Lock()
+	defer context.Unlock()
+
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader.Abort()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// Cleanup does partial shutdown of context
+func (context *AptlyContext) Cleanup() {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// NewContext initializes context with default settings
+func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
+	var err error
+
+	context := &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return nil, err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return nil, err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return context, nil
+}

database/leveldb.go

@@ -0,0 +1,261 @@
+// Package database provides KV database for meta-information
+package database
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// Errors for Storage
+var (
+	ErrNotFound = errors.New("key not found")
+)
+
+// StorageProcessor is a function to process one single storage entry
+type StorageProcessor func(key []byte, value []byte) error
+
+// Storage is an interface to KV storage
+type Storage interface {
+	CreateTemporary() (Storage, error)
+	Get(key []byte) ([]byte, error)
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+	HasPrefix(prefix []byte) bool
+	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
+	KeysByPrefix(prefix []byte) [][]byte
+	FetchByPrefix(prefix []byte) [][]byte
+	Close() error
+	ReOpen() error
+	StartBatch()
+	FinishBatch() error
+	CompactDB() error
+	Drop() error
+}
+
+type levelDB struct {
+	path  string
+	db    *leveldb.DB
+	batch *leveldb.Batch
+}
+
+// Check interface
+var (
+	_ Storage = &levelDB{}
+)
+
+func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
+	o := &opt.Options{
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
+	}
+
+	if throttleCompaction {
+		o.CompactionL0Trigger = 32
+		o.WriteL0PauseTrigger = 96
+		o.WriteL0SlowdownTrigger = 64
+	}
+
+	return leveldb.OpenFile(path, o)
+}
+
+// OpenDB opens (creates) LevelDB database
+func OpenDB(path string) (Storage, error) {
+	db, err := internalOpen(path, false)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDB{db: db, path: path}, nil
+}
+
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := storage.OpenFile(path, false)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}
+
+// CreateTemporary creates new DB of the same type in temp dir
+func (l *levelDB) CreateTemporary() (Storage, error) {
+	tempdir, err := ioutil.TempDir("", "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := internalOpen(tempdir, true)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDB{db: db, path: tempdir}, nil
+}
+
+// Get key value from database
+func (l *levelDB) Get(key []byte) ([]byte, error) {
+	value, err := l.db.Get(key, nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	return value, nil
+}
+
+// Put saves key to database, if key has the same value in DB already, it is not saved
+func (l *levelDB) Put(key []byte, value []byte) error {
+	if l.batch != nil {
+		l.batch.Put(key, value)
+		return nil
+	}
+	old, err := l.db.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Equal(old, value) {
+			return nil
+		}
+	}
+	return l.db.Put(key, value, nil)
+}
+
+// Delete removes key from DB
+func (l *levelDB) Delete(key []byte) error {
+	if l.batch != nil {
+		l.batch.Delete(key)
+		return nil
+	}
+	return l.db.Delete(key, nil)
+}
+
+// KeysByPrefix returns all keys that start with prefix
+func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		key := iterator.Key()
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, keyc)
+	}
+
+	return result
+}
+
+// FetchByPrefix returns all values with keys that start with prefix
+func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		val := iterator.Value()
+		valc := make([]byte, len(val))
+		copy(valc, val)
+		result = append(result, valc)
+	}
+
+	return result
+}
+
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (l *levelDB) HasPrefix(prefix []byte) bool {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (l *levelDB) ProcessByPrefix(prefix []byte, proc StorageProcessor) error {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		err := proc(iterator.Key(), iterator.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Close finishes DB work
+func (l *levelDB) Close() error {
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
+}
+
+// Reopen tries to re-open the database
+func (l *levelDB) ReOpen() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path, false)
+	return err
+}
+
+// StartBatch starts batch processing of keys
+//
+// All subsequent Get, Put and Delete would work on batch
+func (l *levelDB) StartBatch() {
+	if l.batch != nil {
+		panic("batch already started")
+	}
+	l.batch = new(leveldb.Batch)
+}
+
+// FinishBatch finalizes the batch, saving operations
+func (l *levelDB) FinishBatch() error {
+	if l.batch == nil {
+		panic("no batch")
+	}
+	err := l.db.Write(l.batch, nil)
+	l.batch = nil
+	return err
+}
+
+// CompactDB compacts database by merging layers
+func (l *levelDB) CompactDB() error {
+	return l.db.CompactRange(util.Range{})
+}
+
+// Drop removes all the DB files (DANGEROUS!)
+func (l *levelDB) Drop() error {
+	if l.db != nil {
+		return errors.New("DB is still open")
+	}
+
+	return os.RemoveAll(l.path)
+}

database/leveldb_test.go

@@ -0,0 +1,232 @@
+package database
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type LevelDBSuite struct {
+	path string
+	db   Storage
+}
+
+var _ = Suite(&LevelDBSuite{})
+
+func (s *LevelDBSuite) SetUpTest(c *C) {
+	var err error
+
+	s.path = c.MkDir()
+	s.db, err = OpenDB(s.path)
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TearDownTest(c *C) {
+	err := s.db.Close()
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TestRecoverDB(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Check(err, IsNil)
+
+	err = s.db.Close()
+	c.Check(err, IsNil)
+
+	err = RecoverDB(s.path)
+	c.Check(err, IsNil)
+
+	s.db, err = OpenDB(s.path)
+	c.Check(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *LevelDBSuite) TestGetPut(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	_, err := s.db.Get(key)
+	c.Assert(err, ErrorMatches, "key not found")
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *LevelDBSuite) TestTemporaryDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	temp, err := s.db.CreateTemporary()
+	c.Assert(err, IsNil)
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, false)
+
+	err = temp.Put(key, value)
+	c.Assert(err, IsNil)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, true)
+
+	c.Assert(temp.Close(), IsNil)
+	c.Assert(temp.Drop(), IsNil)
+}
+
+func (s *LevelDBSuite) TestDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+
+	_, err = s.db.Get(key)
+	c.Assert(err, ErrorMatches, "key not found")
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+}
+
+func (s *LevelDBSuite) TestByPrefix(c *C) {
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return ErrNotFound
+	}), Equals, ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return ErrNotFound
+	}), IsNil)
+
+	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+}
+
+func (s *LevelDBSuite) TestHasPrefix(c *C) {
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
+func (s *LevelDBSuite) TestBatch(c *C) {
+	var (
+		key    = []byte("key")
+		key2   = []byte("key2")
+		value  = []byte("value")
+		value2 = []byte("value2")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	s.db.StartBatch()
+	s.db.Put(key2, value2)
+	s.db.Delete(key)
+
+	v, err := s.db.Get(key)
+	c.Check(err, IsNil)
+	c.Check(v, DeepEquals, value)
+
+	_, err = s.db.Get(key2)
+	c.Check(err, ErrorMatches, "key not found")
+
+	err = s.db.FinishBatch()
+	c.Check(err, IsNil)
+
+	v2, err := s.db.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = s.db.Get(key)
+	c.Check(err, ErrorMatches, "key not found")
+
+	c.Check(func() { s.db.FinishBatch() }, Panics, "no batch")
+
+	s.db.StartBatch()
+	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
+}
+
+func (s *LevelDBSuite) TestCompactDB(c *C) {
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+
+	c.Check(s.db.CompactDB(), IsNil)
+}
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.ReOpen()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/changes.go

@@ -2,13 +2,14 @@ package deb
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/utils"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // Changes is a result of .changes file parsing
@@ -179,11 +180,29 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 	var binaryQuery PackageQuery
 	if len(c.Binary) > 0 {
 		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		// matching debug ddeb packages, they're not present in the Binary field
+		var ddebQuery PackageQuery
+		ddebQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
+
 		for _, binary := range c.Binary[1:] {
 			binaryQuery = &OrQuery{
 				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
 				R: binaryQuery,
 			}
+			ddebQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", binary)},
+				R: ddebQuery,
+			}
+		}
+
+		ddebQuery = &AndQuery{
+			L: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: c.Source},
+			R: ddebQuery,
+		}
+
+		binaryQuery = &OrQuery{
+			L: binaryQuery,
+			R: ddebQuery,
 		}
 
 		binaryQuery = &AndQuery{

deb/changes_test.go

@@ -1,9 +1,10 @@
 package deb
 
 import (
-	. "gopkg.in/check.v1"
 	"os"
 	"path/filepath"
+
+	. "gopkg.in/check.v1"
 )
 
 type ChangesSuite struct {
@@ -54,7 +55,7 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 	c.Check(err, IsNil)
 
 	c.Check(q.String(), Equals,
-		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), ((Name (= calamares-dbg)) | (Name (= calamares)))))")
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
 }
 
 var changesFile = `Format: 1.8

deb/collections.go

@@ -1,8 +1,9 @@
 package deb
 
 import (
-	"github.com/smira/aptly/database"
 	"sync"
+
+	"github.com/smira/aptly/database"
 )
 
 // CollectionFactory is a single place to generate all desired collections
@@ -21,6 +22,13 @@ func NewCollectionFactory(db database.Storage) *CollectionFactory {
 	return &CollectionFactory{Mutex: &sync.Mutex{}, db: db}
 }
 
+// TemporaryDB creates new temporary DB
+//
+// DB should be closed/droped after being used
+func (factory *CollectionFactory) TemporaryDB() (database.Storage, error) {
+	return factory.db.CreateTemporary()
+}
+
 // PackageCollection returns (or creates) new PackageCollection
 func (factory *CollectionFactory) PackageCollection() *PackageCollection {
 	factory.Lock()

deb/contents.go

@@ -0,0 +1,135 @@
+package deb
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/go-uuid/uuid"
+)
+
+// ContentsIndex calculates mapping from files to packages, with sorting and aggregation
+type ContentsIndex struct {
+	db     database.Storage
+	prefix []byte
+}
+
+// NewContentsIndex creates empty ContentsIndex
+func NewContentsIndex(db database.Storage) *ContentsIndex {
+	return &ContentsIndex{
+		db:     db,
+		prefix: []byte(uuid.New()),
+	}
+}
+
+// Push adds package to contents index, calculating package contents as required
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) error {
+	contents := p.Contents(packagePool)
+	qualifiedName := []byte(p.QualifiedName())
+
+	for _, path := range contents {
+		// for performance reasons we only write to leveldb during push.
+		// merging of qualified names per path will be done in WriteTo
+		err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Empty checks whether index contains no packages
+func (index *ContentsIndex) Empty() bool {
+	return !index.db.HasPrefix(index.prefix)
+}
+
+// WriteTo dumps sorted mapping of files to qualified package names
+func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
+	// For performance reasons push method wrote on key per path and package
+	// in this method we now need to merge all packages which have the same path
+	// and write it to contents index file
+
+	var n int64
+
+	nn, err := fmt.Fprintf(w, "%s %s\n", "FILE", "LOCATION")
+	n += int64(nn)
+	if err != nil {
+		return n, err
+	}
+
+	prefixLen := len(index.prefix)
+
+	var (
+		currentPath []byte
+		currentPkgs [][]byte
+	)
+
+	err = index.db.ProcessByPrefix(index.prefix, func(key []byte, value []byte) error {
+		// cut prefix
+		key = key[prefixLen:]
+
+		i := bytes.Index(key, []byte{0})
+		if i == -1 {
+			return errors.New("corrupted index entry")
+		}
+
+		path := key[:i]
+		pkg := key[i+1:]
+
+		if !bytes.Equal(path, currentPath) {
+			if currentPath != nil {
+				nn, err = w.Write(append(currentPath, ' '))
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+
+				nn, err = w.Write(bytes.Join(currentPkgs, []byte{','}))
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+
+				nn, err = w.Write([]byte{'\n'})
+				n += int64(nn)
+				if err != nil {
+					return err
+				}
+			}
+
+			currentPath = append([]byte(nil), path...)
+			currentPkgs = nil
+		}
+
+		currentPkgs = append(currentPkgs, append([]byte(nil), pkg...))
+
+		return nil
+	})
+
+	if err != nil {
+		return n, err
+	}
+
+	if currentPath != nil {
+		nn, err = w.Write(append(currentPath, ' '))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+
+		nn, err = w.Write(bytes.Join(currentPkgs, []byte{','}))
+		n += int64(nn)
+		if err != nil {
+			return n, err
+		}
+
+		nn, err = w.Write([]byte{'\n'})
+		n += int64(nn)
+	}
+
+	return n, err
+}

deb/deb.go

@@ -2,16 +2,20 @@ package deb
 
 import (
 	"archive/tar"
+	"bufio"
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
-	"github.com/mkrautz/goar"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/go-xz"
-	"github.com/smira/lzma"
 	"io"
 	"os"
 	"strings"
+
+	"github.com/h2non/filetype/matchers"
+	"github.com/mkrautz/goar"
+
+	"github.com/smira/aptly/utils"
+	"github.com/smira/go-xz"
+	"github.com/smira/lzma"
 )
 
 // GetControlFileFromDeb reads control file from deb package
@@ -119,29 +123,41 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 		}
 
 		if strings.HasPrefix(header.Name, "data.tar") {
+			bufReader := bufio.NewReader(library)
+			signature, err := bufReader.Peek(270)
+
+			var isTar bool
+			if err == nil {
+				isTar = matchers.Tar(signature)
+			}
+
 			var tarInput io.Reader
 
 			switch header.Name {
 			case "data.tar":
-				tarInput = library
+				tarInput = bufReader
 			case "data.tar.gz":
-				ungzip, err := gzip.NewReader(library)
-				if err != nil {
-					return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+				if isTar {
+					tarInput = bufReader
+				} else {
+					ungzip, err := gzip.NewReader(bufReader)
+					if err != nil {
+						return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+					}
+					defer ungzip.Close()
+					tarInput = ungzip
 				}
-				defer ungzip.Close()
-				tarInput = ungzip
 			case "data.tar.bz2":
-				tarInput = bzip2.NewReader(library)
+				tarInput = bzip2.NewReader(bufReader)
 			case "data.tar.xz":
-				unxz, err := xz.NewReader(library)
+				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
 					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
 				}
 				defer unxz.Close()
 				tarInput = unxz
 			case "data.tar.lzma":
-				unlzma := lzma.NewReader(library)
+				unlzma := lzma.NewReader(bufReader)
 				defer unlzma.Close()
 				tarInput = unlzma
 			default:

deb/deb_test.go

@@ -1,10 +1,11 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
 	"path/filepath"
 	"runtime"
 
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 

deb/graph.go

@@ -0,0 +1,138 @@
+package deb
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/awalterschulze/gographviz"
+)
+
+// BuildGraph generates graph contents from aptly object database
+func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz.Interface, error) {
+	var err error
+
+	graph := gographviz.NewEscape()
+	graph.SetDir(true)
+	graph.SetName("aptly")
+
+	var labelStart string
+	var labelEnd string
+
+	switch layout {
+	case "vertical":
+		graph.AddAttr("aptly", "rankdir", "LR")
+		labelStart = ""
+		labelEnd = ""
+	case "horizontal":
+		fallthrough
+	default:
+		labelStart = "{"
+		labelEnd = "}"
+	}
+
+	existingNodes := map[string]bool{}
+
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
+		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkgoldenrod1",
+			"label": fmt.Sprintf("%sMirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d%s", labelStart, repo.Name, repo.ArchiveRoot,
+				repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages(), labelEnd),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
+		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "mediumseagreen",
+			"label": fmt.Sprintf("%sRepo %s|comment: %s|pkgs: %d%s", labelStart,
+				repo.Name, repo.Comment, repo.NumPackages(), labelEnd),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		existingNodes[snapshot.UUID] = true
+		return nil
+	})
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		description := snapshot.Description
+		if snapshot.SourceKind == "repo" {
+			description = "Snapshot from repo"
+		}
+
+		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "cadetblue1",
+			"label": fmt.Sprintf("%sSnapshot %s|%s|pkgs: %d%s", labelStart,
+				snapshot.Name, description, snapshot.NumPackages(), labelEnd),
+		})
+
+		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+			for _, uuid := range snapshot.SourceIDs {
+				_, exists := existingNodes[uuid]
+				if exists {
+					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+		graph.AddNode("aptly", repo.UUID, map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkolivegreen1",
+			"label": fmt.Sprintf("%sPublished %s/%s|comp: %s|arch: %s%s", labelStart,
+				repo.Prefix, repo.Distribution, strings.Join(repo.Components(), " "),
+				strings.Join(repo.Architectures, ", "), labelEnd),
+		})
+
+		for _, uuid := range repo.Sources {
+			_, exists := existingNodes[uuid]
+			if exists {
+				graph.AddEdge(uuid, repo.UUID, true, nil)
+			}
+		}
+
+		return nil
+	})
+
+	return graph, nil
+}