Merge pull request #847 from smira/fix-repo-name

Fix repo name in release script

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.gitignore

@@ -22,13 +22,22 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
-coverage*.out
+coverage.txt
 
 *.pyc
 
-_vendor/
+xc-out/
+root/
 
-gen
 man/aptly.1.html
-man/aptly.1.ronn
+man/aptly.1.ronn
+
+.goxc.local.json
+
+system/env/
+
+# created by make build for release artifacts
+build/
+
+pgp/keyrings/aptly2*.gpg
+pgp/keyrings/aptly2*.gpg~

.golangci.yml

@@ -0,0 +1,22 @@
+run:
+  tests: false
+
+
+linters:
+  enable-all: false
+  disable-all: true
+  enable:
+   - govet
+   - golint
+   - gofmt
+   - deadcode
+   - goimports
+   - misspell
+   - ineffassign
+   - staticcheck
+   - varcheck
+   - structcheck
+   - maligned
+   - vetshadow
+   - goconst
+   - interfacer

.goxc.json

@@ -0,0 +1,46 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin",
+		"go-test",
+		"go-vet"
+	],
+    "TasksAppend": [
+		"bintray"
+    ],
+    "TaskSettings": {
+        "debs": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainer-email": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Depends": "bzip2, xz-utils, gnupg, gpgv",
+				"Suggests": "graphviz"
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
+	"Arch": "386 amd64",
+	"Os": "linux darwin freebsd",
+	"MainDirsExclude": "_man,vendor",
+	"BuildSettings": {
+		"LdFlagsXVars": {
+			"Version": "main.Version"
+		}
+	},
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,21 +1,100 @@
+dist: trusty
+sudo: required
+
 language: go
 
-go:
-  - 1.1
-  - 1.2.1
-  - 1.3
-  - tip
+go_import_path: github.com/aptly-dev/aptly
+
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+    - gnupg2
+    - gpgv2
 
 env:
   global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
+    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+    - NO_FTP_ACCESS: "yes"
+    - BOTO_CONFIG: /dev/null
+
+matrix:
+  allow_failures:
+    - go: master
+      env: RUN_LONG_TESTS=no
+  fast_finish: true
+  include:
+    - go: 1.10.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.11.x
+      env: RUN_LONG_TESTS=yes
+    - go: 1.12.x
+      env:
+        - RUN_LONG_TESTS=yes
+        - DEPLOY_BINARIES=yes
+        - APTLY_USER=aptly
+        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
+    - go: master
+      env: RUN_LONG_TESTS=no
+
+before_install:
+  - virtualenv system/env
+  - . system/env/bin/activate
+  - pip install six packaging appdirs
+  - pip install -U pip setuptools
+  - pip install -r system/requirements.txt
+  - make version
 
 install:
   - make prepare
 
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
 
-script: make travis
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false
 
-matrix:
-    allow_failures:
-        - go: tip
+before_deploy:
+  - make release
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
+    file_glob: true
+    file: build/*
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: s3
+    access_key_id:
+      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
+    secret_access_key:
+      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
+    bucket: aptly-nightly
+    skip_cleanup: true
+    acl: public-read
+    local_dir: build
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh nightly
+    skip_cleanup: true
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh release
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -3,4 +3,35 @@ List of contributors, in chronological order:
 * Andrey Smirnov (https://github.com/smira)
 * Sebastien Binet (https://github.com/sbinet)
 * Ryan Uber (https://github.com/ryanuber)
-* Simon Aquino (https://github.com/simonaquino)
+* Simon Aquino (https://github.com/queeno)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -0,0 +1,239 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+
+### List of Repositories
+
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly/info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Your First Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+## Development Setup
+
+This section describes local setup to start contributing to aptly source.
+
+### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.
+
+### Forking and Cloning
+
+As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
+
+    mkdir -p ~/go/src/github.com/aptly-dev
+    cd ~/go/src/github.com/aptly-dev
+    git clone git@github.com:aptly-dev/aptly.git
+    cd aptly
+
+For main repo under your GitHub user and add it as another Git remote:
+
+    git remote add <user> git@github.com:<user>/aptly.git
+
+That way you can continue to build project as is (you don't need to adjust import paths), but you would need
+to specify your remote name when pushing branches:
+
+    git push <user> <your-branch>
+
+### Dependencies
+
+You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+
+    make prepare dev system/env
+
+This is usually one-time action.
+
+### Building
+
+If you want to build aptly binary from your current source tree, run:
+
+    make install
+
+This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
+
+    aptly
+
+Or, if it's not on your path:
+
+    ~/go/bin/aptly
+
+### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### Style Checks
+
+Style checks could be run with:
+
+    make check
+
+aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
+for the linter could be found in [.golangci.yml](.golangci.yml) file.
+
+Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
+
+### Vendored Code
+
+aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
+repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
+packages over packages in `$GOPATH`.
+
+If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
+Usually all you need is `dep ensure` or `dep ensure -update`.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash and Zsh Completion
+
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.
+
+## Design
+
+This section requires future work.
+
+*TBD*
+
+### Database
+
+### Package Pool
+
+### Package
+
+### PackageList, PackageRefList
+
+### LocalRepo, RemoteRepo, Snapshot
+
+### PublishedRepository
+
+### Context
+
+### Collections, CollectionFactory

Gomfile

@@ -1,24 +0,0 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/mitchellh/goamz/s3', :commit => '55f224c07975fddef9d2116600c664e30df3d594'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
-gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '9888007'
-gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
-gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
-
-group :test do
-    gom 'launchpad.net/gocheck'
-end
-
-group :development do
-   gom 'github.com/golang/lint/golint'
-   gom 'github.com/mattn/goveralls'
-   gom 'github.com/axw/gocov/gocov'
-   gom 'code.google.com/p/go.tools/cmd/cover'
-end

Gopkg.lock

@@ -0,0 +1,404 @@
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
+
+[[projects]]
+  digest = "1:e8777c437e157121465a08c05304d5847ae70b5683ef4afeb723b3c9e5e3bc67"
+  name = "github.com/AlekSi/pointer"
+  packages = ["."]
+  pruneopts = ""
+  revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:218a66570875d243938fff08f17ec03e79b18c138d0845b2428bcb4929ffa4fe"
+  name = "github.com/DisposaBoy/JsonConfigReader"
+  packages = ["."]
+  pruneopts = ""
+  revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
+
+[[projects]]
+  digest = "1:658ab137074ef3d1216e01cbe166eb171a1bd77daeae0b686ab1ae184a9a5ec9"
+  name = "github.com/awalterschulze/gographviz"
+  packages = [
+    ".",
+    "ast",
+    "parser",
+    "scanner",
+    "token"
+  ]
+  pruneopts = ""
+  revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
+  version = "v1.0"
+
+[[projects]]
+  digest = "1:a72e35a51b628d148241720897de644fd6d3cea45d8489dae9d373ced5dfc302"
+  name = "github.com/aws/aws-sdk-go"
+  packages = [
+    "aws",
+    "aws/awserr",
+    "aws/awsutil",
+    "aws/client",
+    "aws/client/metadata",
+    "aws/corehandlers",
+    "aws/credentials",
+    "aws/credentials/ec2rolecreds",
+    "aws/credentials/endpointcreds",
+    "aws/credentials/stscreds",
+    "aws/defaults",
+    "aws/ec2metadata",
+    "aws/endpoints",
+    "aws/request",
+    "aws/session",
+    "aws/signer/v4",
+    "internal/sdkio",
+    "internal/sdkrand",
+    "internal/shareddefaults",
+    "private/protocol",
+    "private/protocol/query",
+    "private/protocol/query/queryutil",
+    "private/protocol/rest",
+    "private/protocol/restxml",
+    "private/protocol/xml/xmlutil",
+    "service/s3",
+    "service/sts"
+  ]
+  pruneopts = ""
+  revision = "a72204b9bf8d48230ee0fe8995613b394c66f2da"
+  version = "v1.13.31"
+
+[[projects]]
+  digest = "1:b9922c7da8a89c973758c03bde497017567f785b49bd253402a3587abec1e53d"
+  name = "github.com/cheggaaa/pb"
+  packages = ["."]
+  pruneopts = ""
+  revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
+  version = "v1.0.10"
+
+[[projects]]
+  branch = "master"
+  digest = "1:1120f960f5c334f0f94bad29eefaf73d52d226893369693686148f66c1993f15"
+  name = "github.com/gin-contrib/sse"
+  packages = ["."]
+  pruneopts = ""
+  revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
+
+[[projects]]
+  digest = "1:348ceb76f2ac958e541e4ba3190484b68df28c38ac9720ed4ef8d36af69ce52e"
+  name = "github.com/gin-gonic/gin"
+  packages = [
+    ".",
+    "binding",
+    "render"
+  ]
+  pruneopts = ""
+  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
+  version = "v1.2"
+
+[[projects]]
+  digest = "1:fbed9ba4076145ae05ef7deeb14d49565e112d6c283a8c4304fa3b7be785fa5e"
+  name = "github.com/go-ini/ini"
+  packages = ["."]
+  pruneopts = ""
+  revision = "1730955e3146956d6a087861380f9b4667ed5071"
+  version = "v1.26.0"
+
+[[projects]]
+  branch = "master"
+  digest = "1:27854310d59099f8dcc61dd8af4a69f0a3597f001154b2fb4d1c41baf2e31ec1"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  pruneopts = ""
+  revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
+
+[[projects]]
+  branch = "master"
+  digest = "1:09307dfb1aa3f49a2bf869dcfa4c6c06ecd3c207221bd1c1a1141f0e51f209eb"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  pruneopts = ""
+  revision = "553a641470496b2327abcac10b36396bd98e45c9"
+
+[[projects]]
+  digest = "1:6ee50e0ace655f26787c120dbce51b3185c470f475c9e57d0dcada8ebb115004"
+  name = "github.com/h2non/filetype"
+  packages = ["matchers"]
+  pruneopts = ""
+  revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb"
+  version = "v1.0.5"
+
+[[projects]]
+  branch = "master"
+  digest = "1:64b78d98b8956492576911baf6a1e3499816d4575e485d12792e4abe7d8b6c46"
+  name = "github.com/jlaffaye/ftp"
+  packages = ["."]
+  pruneopts = ""
+  revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
+
+[[projects]]
+  digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e"
+  name = "github.com/jmespath/go-jmespath"
+  packages = ["."]
+  pruneopts = ""
+  revision = "0b12d6b5"
+
+[[projects]]
+  branch = "master"
+  digest = "1:c728183dd470c8bb2d1775edc641a81ec25f5e3804b8b0536e99f71df17a13a6"
+  name = "github.com/kjk/lzma"
+  packages = ["."]
+  pruneopts = ""
+  revision = "3fd93898850d5252457e48c1b3d5e1510597280b"
+
+[[projects]]
+  digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
+  name = "github.com/mattn/go-isatty"
+  packages = ["."]
+  pruneopts = ""
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
+
+[[projects]]
+  digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = ""
+  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
+  version = "v0.0.2"
+
+[[projects]]
+  digest = "1:fb8502ed69803c4cd97b1def119db73869ec2f7e2b408eb9c1b8276df6f05b3e"
+  name = "github.com/mattn/go-shellwords"
+  packages = ["."]
+  pruneopts = ""
+  revision = "005a0944d84452842197c2108bd9168ced206f78"
+  version = "v1.0.2"
+
+[[projects]]
+  branch = "master"
+  digest = "1:3ba28ef4fbbf8d099c6227b698a6ceae28f8a071e12f06839c12c4c0003b7554"
+  name = "github.com/mkrautz/goar"
+  packages = ["."]
+  pruneopts = ""
+  revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
+
+[[projects]]
+  branch = "master"
+  digest = "1:d33ce379780d7c43405b9251289493cabada82f6bf9ab35eea6915d04f6ac8e0"
+  name = "github.com/mxk/go-flowrate"
+  packages = ["flowrate"]
+  pruneopts = ""
+  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
+
+[[projects]]
+  branch = "master"
+  digest = "1:428d8af27f534ed06d783b03d477124796de06aa86402777cd2b494c64278da5"
+  name = "github.com/ncw/swift"
+  packages = [
+    ".",
+    "swifttest"
+  ]
+  pruneopts = ""
+  revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
+
+[[projects]]
+  branch = "master"
+  digest = "1:beeb9206cc21cfeb113066c3dcf4bbb0ba304d73dd441f3244721566f51f44e6"
+  name = "github.com/pborman/uuid"
+  packages = ["."]
+  pruneopts = ""
+  revision = "c65b2f87fee37d1c7854c9164a450713c28d50cd"
+
+[[projects]]
+  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  pruneopts = ""
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
+
+[[projects]]
+  branch = "master"
+  digest = "1:c6e71685a14409c3a1fdbecef044ee6447c3887de7c2bb025ee12f9a8a368960"
+  name = "github.com/smira/commander"
+  packages = ["."]
+  pruneopts = ""
+  revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
+
+[[projects]]
+  branch = "master"
+  digest = "1:a75eb2e6e718988505f704d651eeb4734f0f0b580f761aa58b4a5b0afe585091"
+  name = "github.com/smira/flag"
+  packages = ["."]
+  pruneopts = ""
+  revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
+
+[[projects]]
+  branch = "master"
+  digest = "1:cd3b8057abbb453404cbef569951a7359986eedfc0a887f9bc5efff8d02e1760"
+  name = "github.com/smira/go-aws-auth"
+  packages = ["."]
+  pruneopts = ""
+  revision = "8b73995fd8d1d82519c7037ee243a497552cd54d"
+
+[[projects]]
+  branch = "master"
+  digest = "1:7dd59b3536a07651baf394466da3b8a20bd3ed9ce1984c7583fdf2412fdcfea9"
+  name = "github.com/smira/go-ftp-protocol"
+  packages = ["protocol"]
+  pruneopts = ""
+  revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
+
+[[projects]]
+  branch = "master"
+  digest = "1:55fe1086100b305a5562e9f1e6825ef97c49e82331cb0f055b2069f7c7ebd469"
+  name = "github.com/smira/go-xz"
+  packages = ["."]
+  pruneopts = ""
+  revision = "0c531f070014e218b21f3cfca801cc992d52726d"
+
+[[projects]]
+  branch = "master"
+  digest = "1:175bc8d87d54849b9b9fc6cd473c5830e090fbf3c2f0ca71c7642a697e5d9237"
+  name = "github.com/syndtr/goleveldb"
+  packages = [
+    "leveldb",
+    "leveldb/cache",
+    "leveldb/comparer",
+    "leveldb/errors",
+    "leveldb/filter",
+    "leveldb/iterator",
+    "leveldb/journal",
+    "leveldb/memdb",
+    "leveldb/opt",
+    "leveldb/storage",
+    "leveldb/table",
+    "leveldb/util"
+  ]
+  pruneopts = ""
+  revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
+
+[[projects]]
+  digest = "1:b9bca27d5fbe4ad1a1802706629580ce923bb14255c51375480dff1bebcbb8b2"
+  name = "github.com/ugorji/go"
+  packages = ["codec"]
+  pruneopts = ""
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[projects]]
+  branch = "master"
+  digest = "1:2d8078b329a80bf4b71537dcd30853ff89fcafb611c94b2dfdb3d43ef1f23a1a"
+  name = "github.com/wsxiaoys/terminal"
+  packages = ["color"]
+  pruneopts = ""
+  revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
+
+[[projects]]
+  branch = "master"
+  digest = "1:51aff2b69272cb95713b74675c0817e0a0caadc20337a8d9115bc68bf1105280"
+  name = "golang.org/x/crypto"
+  packages = [
+    "cast5",
+    "openpgp",
+    "openpgp/armor",
+    "openpgp/clearsign",
+    "openpgp/elgamal",
+    "openpgp/errors",
+    "openpgp/packet",
+    "openpgp/s2k",
+    "ssh/terminal"
+  ]
+  pruneopts = ""
+  revision = "b2aa35443fbc700ab74c586ae79b81c171851023"
+
+[[projects]]
+  branch = "master"
+  digest = "1:267aa6124260d0e48d1ebf8bb84dce97caa133406b3caabc076cfc7a9e85ffe4"
+  name = "golang.org/x/sys"
+  packages = [
+    "unix",
+    "windows"
+  ]
+  pruneopts = ""
+  revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e"
+
+[[projects]]
+  branch = "v1"
+  digest = "1:e75566abfb876e81f00290ec153ff994c33bf8886134c1a38a9a9df5c15a2045"
+  name = "gopkg.in/check.v1"
+  packages = ["."]
+  pruneopts = ""
+  revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
+
+[[projects]]
+  digest = "1:dd549e360e5a8f982a28c2bcbe667307ceffe538ed9afc7c965524f1ac285b3f"
+  name = "gopkg.in/go-playground/validator.v8"
+  packages = ["."]
+  pruneopts = ""
+  revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
+  version = "v8.18.2"
+
+[[projects]]
+  digest = "1:0e886d5a5845f58255fb57bcb4a6aae7e1f9dd7c53defe7b87d57dd4cd127545"
+  name = "gopkg.in/h2non/filetype.v1"
+  packages = ["types"]
+  pruneopts = ""
+  revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
+  version = "v1.0.1"
+
+[[projects]]
+  branch = "v2"
+  digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  pruneopts = ""
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/AlekSi/pointer",
+    "github.com/DisposaBoy/JsonConfigReader",
+    "github.com/awalterschulze/gographviz",
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/awserr",
+    "github.com/aws/aws-sdk-go/aws/corehandlers",
+    "github.com/aws/aws-sdk-go/aws/credentials",
+    "github.com/aws/aws-sdk-go/aws/request",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/s3",
+    "github.com/cheggaaa/pb",
+    "github.com/gin-gonic/gin",
+    "github.com/h2non/filetype/matchers",
+    "github.com/kjk/lzma",
+    "github.com/mattn/go-shellwords",
+    "github.com/mkrautz/goar",
+    "github.com/mxk/go-flowrate/flowrate",
+    "github.com/ncw/swift",
+    "github.com/ncw/swift/swifttest",
+    "github.com/pborman/uuid",
+    "github.com/pkg/errors",
+    "github.com/smira/commander",
+    "github.com/smira/flag",
+    "github.com/smira/go-aws-auth",
+    "github.com/smira/go-ftp-protocol/protocol",
+    "github.com/smira/go-xz",
+    "github.com/syndtr/goleveldb/leveldb",
+    "github.com/syndtr/goleveldb/leveldb/filter",
+    "github.com/syndtr/goleveldb/leveldb/opt",
+    "github.com/syndtr/goleveldb/leveldb/storage",
+    "github.com/syndtr/goleveldb/leveldb/util",
+    "github.com/ugorji/go/codec",
+    "github.com/wsxiaoys/terminal/color",
+    "golang.org/x/crypto/openpgp",
+    "golang.org/x/crypto/openpgp/armor",
+    "golang.org/x/crypto/openpgp/clearsign",
+    "golang.org/x/crypto/openpgp/errors",
+    "golang.org/x/crypto/openpgp/packet",
+    "golang.org/x/crypto/ssh/terminal",
+    "golang.org/x/sys/unix",
+    "gopkg.in/check.v1"
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -0,0 +1,28 @@
+
+[[override]]
+  branch = "master"
+  name = "github.com/mkrautz/goar"
+
+[[override]]
+  branch = "master"
+  name = "github.com/smira/go-uuid"
+
+[[override]]
+  branch = "master"
+  name = "github.com/smira/go-xz"
+
+[[override]]
+  name = "github.com/ugorji/go"
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[override]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+
+[[override]]
+  branch = "master"
+  name = "golang.org/x/sys"
+
+[[override]]
+  branch = "v1"
+  name = "gopkg.in/check.v1"

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,85 +1,77 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http query s3 utils
-ALL_PACKAGES=aptly cmd console database deb files http query s3 utils
-BINPATH=$(abspath ./_vendor/bin)
-GOM_ENVIRONMENT=-test
+ifdef TRAVIS_TAG
+	TAG=$(TRAVIS_TAG)
+else
+	TAG="$(shell git describe --tags)"
+endif
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
+PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python
+TESTS?=
+BINPATH?=$(GOPATH)/bin
+RUN_LONG_TESTS?=yes
 
-ifeq ($(GOVERSION), devel)
-TRAVIS_TARGET=coveralls
-GOM_ENVIRONMENT+=-development
-else
-TRAVIS_TARGET=test
-endif
-
-ifeq ($(TRAVIS), true)
-GOM=$(HOME)/gopath/bin/gom
-else
-GOM=gom
-endif
-
-all: test check system-test
+all: test bench check system-test
 
 prepare:
-	go get -u github.com/mattn/gom
-	$(GOM) $(GOM_ENVIRONMENT) install
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.17.1
 
-coverage.out:
-	rm -f coverage.*.out
-	for i in $(PACKAGES); do $(GOM) test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-	rm -f coverage.*.out
+dev:
+	go get -u github.com/golang/dep/...
+	go get -u github.com/laher/goxc
 
-coverage: coverage.out
-	$(GOM) exec go tool cover -html=coverage.out
-	rm -f coverage.out
-
-check:
-	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
-	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
+check: system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	golangci-lint run
+	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
+endif
 
 install:
-	$(GOM) build -o $(BINPATH)/aptly
+	go install -v -ldflags "-X main.Version=$(VERSION)"
 
-system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 go1.3 devel))
-	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+system/env: system/requirements.txt
+ifeq ($(RUN_LONG_TESTS), yes)
+	rm -rf system/env
+	virtualenv system/env
+	system/env/bin/pip install -r system/requirements.txt
 endif
-	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
-travis: $(TRAVIS_TARGET) system-test
+system-test: install system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
+endif
 
 test:
-	$(GOM) test -v ./... -gocheck.v=true
+	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
 
-coveralls: coverage.out
-	$(GOM) exec $(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+bench:
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
+goxc: dev
 	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
 	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	cp completion.d/aptly root/etc/bash_completion.d/
+	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
 	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
+	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
 
-src-package:
-	rm -rf aptly-$(VERSION)
-	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
-	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
-	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
-	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
-	rm -rf aptly-$(VERSION)
+release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
+release: goxc
+	rm -rf build/
+	mkdir -p build/
+	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
 
-.PHONY: coverage.out
+man:
+	make -C man
+
+version:
+	@echo $(VERSION)
+
+.PHONY: man version release goxc

README.rst

@@ -2,15 +2,24 @@
 aptly
 =====
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
-    :target: https://travis-ci.org/smira/aptly
+.. image:: https://api.travis-ci.org/aptly-dev/aptly.svg?branch=master
+    :target: https://travis-ci.org/aptly-dev/aptly
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
+  :target: https://codecov.io/gh/aptly-dev/aptly
+
+.. image:: https://badges.gitter.im/Join Chat.svg
+    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: http://goreportcard.com/badge/aptly-dev/aptly
+    :target: http://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
 Aptly features: ("+" means planned features)
@@ -20,26 +29,26 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
-* publish self-made packages as Debian repositories (+)
+* filter repository by search query, pulling dependencies when required
+* publish self-made packages as Debian repositories
+* REST API for remote access
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------
 
-To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
     deb http://repo.aptly.info/ squeeze main
 
 And import key that is used to sign the release::
 
-    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
-    $ gpg -a --export 2A194991 | sudo apt-key add -
+    $ apt-key adv --keyserver pool.sks-keyservers.net --recv-keys ED75B5A4483DA07C
 
 After that you can install aptly as any other software package::
 
@@ -49,20 +58,61 @@ After that you can install aptly as any other software package::
 Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
 Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
+If you would like to use nightly builds (unstable), please use following repository::
+
+    deb http://repo.aptly.info/ nightly main
+
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.10+ required)::
 
-    go get -u github.com/mattn/gom
-    mkdir -p $GOPATH/src/github.com/smira/aptly
-    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
-    cd $GOPATH/src/github.com/smira/aptly
-    gom -production install
-    gom build -o $GOPATH/bin/aptly
+    mkdir -p $GOPATH/src/github.com/aptly-dev/aptly
+    git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly
+    cd $GOPATH/src/github.com/aptly-dev/aptly
+    make install
 
-Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
-should work as well, but might fail or produce different result (if external libraries got updated).
+Binary would be installed to ``$GOPATH/bin/aptly``.
 
-If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+Contributing
+------------
 
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
+Integrations
+------------
+
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
+
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -2,16 +2,17 @@ package main
 
 import (
 	"fmt"
-	"github.com/smira/aptly/cmd"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"text/template"
+
+	"github.com/aptly-dev/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func allFlags(flags *flag.FlagSet) []*flag.Flag {
@@ -43,22 +44,42 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
 	command.Dispatch(nil)
 
-	_, _File, _, _ := runtime.Caller(0)
-	_File, _ = filepath.Abs(_File)
+	_File, _ := filepath.Abs("./man")
 
 	templ := template.New("man").Funcs(template.FuncMap{
 		"allFlags":    allFlags,
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

api/api.go

@@ -0,0 +1,165 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/gin-gonic/gin"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+type dbRequestKind int
+
+const (
+	acquiredb dbRequestKind = iota
+	releasedb
+)
+
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
+// Flushes all collections which cache in-memory objects
+func flushColections() {
+	// lock everything to eliminate in-progress calls
+	r := context.CollectionFactory().RemoteRepoCollection()
+	r.Lock()
+	defer r.Unlock()
+
+	l := context.CollectionFactory().LocalRepoCollection()
+	l.Lock()
+	defer l.Unlock()
+
+	s := context.CollectionFactory().SnapshotCollection()
+	s.Lock()
+	defer s.Unlock()
+
+	p := context.CollectionFactory().PublishedRepoCollection()
+	p.Lock()
+	defer p.Unlock()
+
+	// all collections locked, flush them
+	context.CollectionFactory().Flush()
+}
+
+// Periodically flushes CollectionFactory to free up memory used by
+// collections, flushing caches.
+//
+// Should be run in goroutine!
+func cacheFlusher() {
+	ticker := time.Tick(15 * time.Minute)
+
+	for {
+		<-ticker
+
+		flushColections()
+	}
+}
+
+// Acquire database lock and release it when not needed anymore.
+//
+// Should be run in a goroutine!
+func acquireDatabase(requests <-chan dbRequest) {
+	clients := 0
+	for request := range requests {
+		var err error
+
+		switch request.kind {
+		case acquiredb:
+			if clients == 0 {
+				err = context.ReOpenDatabase()
+			}
+
+			request.err <- err
+
+			if err == nil {
+				clients++
+			}
+		case releasedb:
+			clients--
+			if clients == 0 {
+				flushColections()
+				err = context.CloseDatabase()
+			} else {
+				err = nil
+			}
+
+			request.err <- err
+		}
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			return
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/files.go

@@ -0,0 +1,186 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.AbortWithError(404, err)
+		} else {
+			c.AbortWithError(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.AbortWithError(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
+
+	factory := context.CollectionFactory()
+
+	factory.RemoteRepoCollection().Lock()
+	defer factory.RemoteRepoCollection().Unlock()
+	factory.LocalRepoCollection().Lock()
+	defer factory.LocalRepoCollection().Unlock()
+	factory.SnapshotCollection().Lock()
+	defer factory.SnapshotCollection().Unlock()
+	factory.PublishedRepoCollection().Lock()
+	defer factory.PublishedRepoCollection().Unlock()
+
+	graph, err := deb.BuildGraph(factory, layout)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/packages.go

@@ -0,0 +1,16 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,380 @@
+package api
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (pgp.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := context.GetSigner()
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	localCollection := context.CollectionFactory().LocalRepoCollection()
+	localCollection.RLock()
+	defer localCollection.RUnlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, context.CollectionFactory())
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution         string
+		Label                string
+		Origin               string
+		NotAutomatic         string
+		ButAutomaticUpgrades string
+		ForceOverwrite       bool
+		SkipContents         *bool
+		Architectures        []string
+		Signing              SigningOptions
+		AcquireByHash        *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var sources []interface{}
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := context.CollectionFactory().SnapshotCollection()
+		snapshotCollection.Lock()
+		defer snapshotCollection.Unlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == deb.SourceLocalRepo {
+		var localRepo *deb.LocalRepo
+
+		localCollection := context.CollectionFactory().LocalRepoCollection()
+		localCollection.Lock()
+		defer localCollection.Unlock()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+	if b.Origin != "" {
+		published.Origin = b.Origin
+	}
+	if b.NotAutomatic != "" {
+		published.NotAutomatic = b.NotAutomatic
+	}
+	if b.ButAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+	}
+	published.Label = b.Label
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
+	duplicate := collection.CheckDuplicate(published)
+	if duplicate != nil {
+		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		c.AbortWithError(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		return
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	err = collection.Add(published)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	c.JSON(201, published)
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipContents   *bool
+		SkipCleanup    *bool
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+		AcquireByHash *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.Lock()
+	defer localRepoCollection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, context.CollectionFactory())
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		if len(b.Snapshots) > 0 {
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				c.AbortWithError(404, err2)
+				return
+			}
+
+			err2 = snapshotCollection.LoadComplete(snapshot)
+			if err2 != nil {
+				c.AbortWithError(500, err2)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+		}
+	} else {
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	err = collection.Update(published)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
+		return
+	}
+
+	if b.SkipCleanup == nil || !*b.SkipCleanup {
+		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+			return
+		}
+	}
+
+	c.JSON(200, published)
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	// published.LoadComplete would touch local repo collection
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.Lock()
+	defer localRepoCollection.Unlock()
+
+	collection := context.CollectionFactory().PublishedRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := collection.Remove(context, storage, prefix, distribution,
+		context.CollectionFactory(), context.Progress(), force, skipCleanup)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/repos.go

@@ -0,0 +1,444 @@
+package api
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Comment             *string
+		DefaultDistribution *string
+		DefaultComponent    *string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	if b.Comment != nil {
+		repo.Comment = *b.Comment
+	}
+	if b.DefaultDistribution != nil {
+		repo.DefaultDistribution = *b.DefaultDistribution
+	}
+	if b.DefaultComponent != nil {
+		repo.DefaultComponent = *b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.RLock()
+	defer snapshotCollection.RUnlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	published := publishedCollection.ByLocalRepo(repo)
+	if len(published) > 0 {
+		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+		if len(snapshots) > 0 {
+			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = collection.Drop(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList())
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.AbortWithError(500, err)
+			}
+			return
+		}
+		err = cb(list, p)
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	verifier := context.GetVerifier()
+
+	var (
+		sources                      []string
+		packageFiles, failedFiles    []string
+		otherFiles                   []string
+		processedFiles, failedFiles2 []string
+		reporter                     = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+		list *deb.PackageList
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
+		return
+	}
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	processedFiles = append(processedFiles, otherFiles...)
+
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
+		return
+	}
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
+		return
+	}
+
+	if !noRemove {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				reporter.Warning("unable to remove file %s: %s", file, err)
+			}
+		}
+
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}
+
+// POST /repos/:name/include/:dir/:file
+func apiReposIncludePackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposIncludePackageFromDir(c)
+}
+
+// POST /repos/:name/include/:dir
+func apiReposIncludePackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
+	acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
+	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
+
+	repoTemplateString := c.Params.ByName("name")
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	var (
+		err                       error
+		verifier                  = context.GetVerifier()
+		sources, changesFiles     []string
+		failedFiles, failedFiles2 []string
+		reporter                  = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.Lock()
+	defer localRepoCollection.Unlock()
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+		repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
+		return
+	}
+
+	if !noRemoveFiles {
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -0,0 +1,119 @@
+package api
+
+import (
+	"net/http"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+)
+
+var context *ctx.AptlyContext
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	router := gin.Default()
+	router.Use(gin.ErrorLogger())
+
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		requests := make(chan dbRequest)
+
+		go acquireDatabase(requests)
+
+		router.Use(func(c *gin.Context) {
+			var err error
+
+			errCh := make(chan error)
+			requests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			defer func() {
+				requests <- dbRequest{releasedb, errCh}
+				err = <-errCh
+				if err != nil {
+					c.AbortWithError(500, err)
+				}
+			}()
+
+			c.Next()
+		})
+
+	} else {
+		go cacheFlusher()
+	}
+
+	root := router.Group("/api")
+
+	{
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,411 @@
+package api
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.RLock()
+	defer collection.RUnlock()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collection := context.CollectionFactory().RemoteRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		c.AbortWithError(409, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.AbortWithError(500, err)
+			return
+		}
+	}
+
+	list := deb.NewPackageList()
+
+	// verify package refs and build package list
+	for _, ref := range b.PackageRefs {
+		var p *deb.Package
+
+		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+		if err != nil {
+			if err == database.ErrNotFound {
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
+			} else {
+				c.AbortWithError(500, err)
+			}
+			return
+		}
+		err = list.Add(p)
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+	}
+
+	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collection := context.CollectionFactory().LocalRepoCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	repo, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, snapshot)
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	_, err = collection.ByName(b.Name)
+	if err == nil {
+		c.AbortWithError(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		return
+	}
+
+	if b.Name != "" {
+		snapshot.Name = b.Name
+	}
+
+	if b.Description != "" {
+		snapshot.Description = b.Description
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	snapshotCollection := context.CollectionFactory().SnapshotCollection()
+	snapshotCollection.Lock()
+	defer snapshotCollection.Unlock()
+
+	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
+	publishedCollection.RLock()
+	defer publishedCollection.RUnlock()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	published := publishedCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		c.AbortWithError(409, fmt.Errorf("unable to drop: snapshot is published"))
+		return
+	}
+
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			c.AbortWithError(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			return
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collection := context.CollectionFactory().SnapshotCollection()
+	collection.Lock()
+	defer collection.Unlock()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList())
+}

aptly-api.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository API
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/api/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
+
+[Install]
+WantedBy=multi-user.target

aptly.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository server
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/commands/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
+
+[Install]
+WantedBy=multi-user.target

aptly/interfaces.go

@@ -3,24 +3,59 @@
 package aptly
 
 import (
-	"github.com/smira/aptly/utils"
+	"context"
 	"io"
+	"os"
+
+	"github.com/aptly-dev/aptly/utils"
 )
 
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
 // PackagePool is asbtraction of package pool storage.
 //
 // PackagePool stores all the package files, deduplicating them.
 type PackagePool interface {
-	// Path returns full path to package file in pool given any name and hash of file contents
-	Path(filename string, hashMD5 string) (string, error)
-	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
-	RelativePath(filename string, hashMD5 string) (string, error)
+	// Verify checks whether file exists in the pool and fills back checksum info
+	//
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
 	FilepathList(progress Progress) ([]string, error)
 	// Remove deletes file in package pool returns its size
 	Remove(path string) (size int64, err error)
-	// Import copies file into package pool
-	Import(path string, hashMD5 string) error
+}
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
 }
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
@@ -34,15 +69,23 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string) error
+	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
-// LocalPublishedStorage is published storage on local filesystem
-type LocalPublishedStorage interface {
+// FileSystemPublishedStorage is published storage on filesystem
+type FileSystemPublishedStorage interface {
 	// PublicPath returns root of public part
 	PublicPath() string
 }
@@ -75,21 +118,26 @@ type Progress interface {
 	Printf(msg string, a ...interface{})
 	// ColoredPrintf does printf in colored way + newline
 	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
 }
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string, result chan<- error)
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
-	// Pause pauses task processing
-	Pause()
-	// Resume resumes task processing
-	Resume()
-	// Shutdown stops downloader after current tasks are finished,
-	// but doesn't process rest of queue
-	Shutdown()
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
+}
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
 }

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
-// Version of aptly
-const Version = "0.7"
+// Version of aptly (filled in at link time)
+var Version string
 
-// Enable debugging features?
+// EnableDebug triggers some debugging features
 const EnableDebug = false

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,127 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/aptly-dev/aptly/api"
+	"github.com/aptly-dev/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+
+		var stat os.FileInfo
+		stat, err = os.Stat(file)
+		shouldRemove := true
+
+		if err == nil && stat.Mode()&os.ModeSocket == os.ModeSocket {
+			shouldRemove = false
+		}
+
+		if err != nil && os.IsNotExist(err) {
+			shouldRemove = false
+		}
+
+		if shouldRemove {
+			err = os.Remove(file)
+			if err != nil {
+				fmt.Printf("Warning: error removing file %s: %s\n", file, err)
+			}
+		}
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -2,13 +2,22 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"time"
+)
+
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
@@ -19,18 +28,52 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 		return
 	}
 
-	err = reflist.ForEach(func(key []byte) error {
-		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err2 != nil {
-			return err2
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	return PrintPackageList(list, "", "  ")
+
+}
+
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
+	if format == "" {
+		return result.ForEachIndexed(func(p *deb.Package) error {
+			context.Progress().Printf(prefix+"%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEachIndexed(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf(prefix+"%s\n", b.String())
+		return nil
+	})
+
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
 	return
 }
 
@@ -46,30 +89,37 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
+			makeCmdConfig(),
 			makeCmdDb(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
+			makeCmdAPI(),
 		},
 	}
 
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -1,321 +1,36 @@
 package cmd
 
 import (
-	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/http"
-	"github.com/smira/aptly/s3"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"time"
 )
 
-// AptlyContext is a common context shared by all commands
-type AptlyContext struct {
-	flags        *flag.FlagSet
-	configLoaded bool
-
-	progress          aptly.Progress
-	downloader        aptly.Downloader
-	database          database.Storage
-	packagePool       aptly.PackagePool
-	publishedStorages map[string]aptly.PublishedStorage
-	collectionFactory *deb.CollectionFactory
-	dependencyOptions int
-	architecturesList []string
-	// Debug features
-	fileCPUProfile *os.File
-	fileMemProfile *os.File
-	fileMemStats   *os.File
-}
-
-var context *AptlyContext
-
-// Check interface
-var _ aptly.PublishedStorageProvider = &AptlyContext{}
-
-// FatalError is type for panicking to abort execution with non-zero
-// exit code and print meaningful explanation
-type FatalError struct {
-	ReturnCode int
-	Message    string
-}
-
-// Fatal panics and aborts execution with exit code 1
-func Fatal(err error) {
-	returnCode := 1
-	if err == commander.ErrFlagError || err == commander.ErrCommandError {
-		returnCode = 2
-	}
-	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
-}
-
-// Config loads and returns current configuration
-func (context *AptlyContext) Config() *utils.ConfigStructure {
-	if !context.configLoaded {
-		var err error
-
-		configLocation := context.flags.Lookup("config").Value.String()
-		if configLocation != "" {
-			err = utils.LoadConfig(configLocation, &utils.Config)
-
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			configLocations := []string{
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-				"/etc/aptly.conf",
-			}
-
-			for _, configLocation := range configLocations {
-				err = utils.LoadConfig(configLocation, &utils.Config)
-				if err == nil {
-					break
-				}
-				if !os.IsNotExist(err) {
-					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-				}
-			}
-
-			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-				utils.SaveConfig(configLocations[0], &utils.Config)
-			}
-		}
-
-		context.configLoaded = true
-
-	}
-	return &utils.Config
-}
-
-// DependencyOptions calculates options related to dependecy handling
-func (context *AptlyContext) DependencyOptions() int {
-	if context.dependencyOptions == -1 {
-		context.dependencyOptions = 0
-		if context.Config().DepFollowSuggests || context.flags.Lookup("dep-follow-suggests").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowSuggests
-		}
-		if context.Config().DepFollowRecommends || context.flags.Lookup("dep-follow-recommends").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowRecommends
-		}
-		if context.Config().DepFollowAllVariants || context.flags.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowAllVariants
-		}
-		if context.Config().DepFollowSource || context.flags.Lookup("dep-follow-source").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowSource
-		}
-	}
-
-	return context.dependencyOptions
-}
-
-// ArchitecturesList returns list of architectures fixed via command line or config
-func (context *AptlyContext) ArchitecturesList() []string {
-	if context.architecturesList == nil {
-		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.flags.Lookup("architectures").Value.String()
-		if optionArchitectures != "" {
-			context.architecturesList = strings.Split(optionArchitectures, ",")
-		}
-	}
-
-	return context.architecturesList
-}
-
-// Progress creates or returns Progress object
-func (context *AptlyContext) Progress() aptly.Progress {
-	if context.progress == nil {
-		context.progress = console.NewProgress()
-		context.progress.Start()
-	}
-
-	return context.progress
-}
-
-// Downloader returns instance of current downloader
-func (context *AptlyContext) Downloader() aptly.Downloader {
-	if context.downloader == nil {
-		var downloadLimit int64
-		limitFlag := context.flags.Lookup("download-limit")
-		if limitFlag != nil {
-			downloadLimit = limitFlag.Value.Get().(int64)
-		}
-		if downloadLimit == 0 {
-			downloadLimit = context.Config().DownloadLimit
-		}
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency,
-			downloadLimit*1024, context.Progress())
-	}
-
-	return context.downloader
-}
-
-// DBPath builds path to database
-func (context *AptlyContext) DBPath() string {
-	return filepath.Join(context.Config().RootDir, "db")
-}
-
-// Database opens and returns current instance of database
-func (context *AptlyContext) Database() (database.Storage, error) {
-	if context.database == nil {
-		var err error
-
-		context.database, err = database.OpenDB(context.DBPath())
-		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
-		}
-	}
-
-	return context.database, nil
-}
-
-// CollectionFactory builds factory producing all kinds of collections
-func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
-	if context.collectionFactory == nil {
-		db, err := context.Database()
-		if err != nil {
-			Fatal(err)
-		}
-		context.collectionFactory = deb.NewCollectionFactory(db)
-	}
-
-	return context.collectionFactory
-}
-
-// PackagePool returns instance of PackagePool
-func (context *AptlyContext) PackagePool() aptly.PackagePool {
-	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.Config().RootDir)
-	}
-
-	return context.packagePool
-}
-
-// PublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
-	publishedStorage, ok := context.publishedStorages[name]
-	if !ok {
-		if name == "" {
-			publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
-		} else if strings.HasPrefix(name, "s3:") {
-			params, ok := context.Config().S3PublishRoots[name[3:]]
-			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
-			}
-
-			var err error
-			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix)
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
-		}
-		context.publishedStorages[name] = publishedStorage
-	}
-
-	return publishedStorage
-}
+var context *ctx.AptlyContext
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if aptly.EnableDebug {
-		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-		if context.fileCPUProfile != nil {
-			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
-			context.fileCPUProfile = nil
-		}
-		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-	}
-	if context.database != nil {
-		context.database.Close()
-	}
-	if context.downloader != nil {
-		context.downloader.Shutdown()
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-	}
+	context.Shutdown()
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	context.Cleanup()
 }
 
 // InitContext initializes context with default settings
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context = &AptlyContext{
-		flags:             flags,
-		dependencyOptions: -1,
-		publishedStorages: map[string]aptly.PublishedStorage{},
+	if context != nil {
+		panic("context already initialized")
 	}
 
-	if aptly.EnableDebug {
-		cpuprofile := flags.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
-			if err != nil {
-				return err
-			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
+	context, err = ctx.NewContext(flags)
 
-		memprofile := flags.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := flags.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
-			}
-
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
-				}
-			}()
-		}
-	}
-
-	return nil
+	return err
+}
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
 }

cmd/db_cleanup.go

@@ -2,10 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
 	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
 )
 
 // aptly db cleanup
@@ -17,31 +19,102 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
+
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	context.CollectionFactory().Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
 		}
 		return nil
 	})
@@ -49,51 +122,104 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
+	context.CollectionFactory().Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
+	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
+		if published.SourceKind != deb.SourceLocalRepo {
+			return nil
+		}
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		if e != nil {
+			return e
+		}
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Substract(existingPackageRefs)
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
+
+	if toDelete.Len() > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			db.StartBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+			})
+			if err != nil {
+				return err
+			}
+
+			err = db.FinishBatch()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	err = db.FinishBatch()
-	if err != nil {
-		return fmt.Errorf("unable to write to DB: %s", err)
-	}
+	context.CollectionFactory().Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
 	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
 		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -112,7 +238,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -122,28 +248,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
-
-		var size, totalSize int64
-		for _, file := range filesToDelete {
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
+			for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
-	err = db.CompactDB()
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -163,5 +304,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/db_recover.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 	"github.com/smira/commander"
 )
 

cmd/graph.go

@@ -2,15 +2,19 @@ package cmd
 
 import (
 	"bytes"
-	"code.google.com/p/gographviz"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strings"
+	"time"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -21,121 +25,14 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	graph := gographviz.NewEscape()
-	graph.SetDir(true)
-	graph.SetName("aptly")
-
-	existingNodes := map[string]bool{}
-
-	fmt.Printf("Loading mirrors...\n")
-
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkgoldenrod1",
-			"label": fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading local repos...\n")
-
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "mediumseagreen",
-			"label": fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages()),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading snapshots...\n")
-
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		existingNodes[snapshot.UUID] = true
-		return nil
-	})
-
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-
-		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
-			description = "Snapshot from repo"
-		}
-
-		graph.AddNode("aptly", snapshot.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "cadetblue1",
-			"label":     fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages()),
-		})
-
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
-			for _, uuid := range snapshot.SourceIDs {
-				_, exists := existingNodes[uuid]
-				if exists {
-					graph.AddEdge(uuid, snapshot.UUID, true, nil)
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading published repos...\n")
-
-	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		graph.AddNode("aptly", repo.UUID, map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkolivegreen1",
-			"label": fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution,
-				strings.Join(repo.Components(), " "), strings.Join(repo.Architectures, ", ")),
-		})
-
-		for _, uuid := range repo.Sources {
-			_, exists := existingNodes[uuid]
-			if exists {
-				graph.AddEdge(uuid, repo.UUID, true, nil)
-			}
-		}
-
-		return nil
-	})
+	layout := context.Flags().Lookup("layout").Value.String()
 
 	fmt.Printf("Generating graph...\n")
+	graph, err := deb.BuildGraph(context.CollectionFactory(), layout)
+
+	if err != nil {
+		return err
+	}
 
 	buf := bytes.NewBufferString(graph.String())
 
@@ -146,9 +43,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -176,15 +80,51 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	defer func() {
+		_ = os.Remove(tempfilename)
+	}()
+
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		if err = viewer.Start(); err == nil {
+			// Wait for a second so that the visualizer has a chance to
+			// open the input file. This needs to be done even if we're
+			// waiting for the visualizer as it can be just a wrapper that
+			// spawns a browser tab and returns right away.
+			defer func(t <-chan time.Time) {
+				<-t
+			}(time.After(time.Second))
+		}
 	}
 
 	return err
 }
 
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
 func makeCmdGraph() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyGraph,
@@ -201,5 +141,9 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
 	return cmd
 }

cmd/mirror.go

@@ -1,20 +1,21 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
-func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if context.Config().GpgDisableVerify || flags.Lookup("ignore-signatures").Value.Get().(bool) {
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
@@ -56,6 +57,7 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorUpdate(),
 			makeCmdMirrorRename(),
 			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
 	}
 }

cmd/mirror_create.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
@@ -16,7 +17,9 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -33,13 +36,16 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.flags.Lookup("filter").Value.String()
-	repo.FilterWithDeps = context.flags.Lookup("filter-with-deps").Value.Get().(bool)
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -48,7 +54,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	verifier, err := getVerifier(context.flags)
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -74,7 +80,7 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
 line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
@@ -89,9 +95,13 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -20,7 +21,12 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
 

cmd/mirror_edit.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/query"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -19,15 +21,34 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	context.flags.Visit(func(flag *flag.Flag) {
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fetchMirror := false
+	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
 			repo.Filter = flag.Value.String()
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
 		}
 	})
 
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -35,6 +56,24 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
+
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
@@ -48,10 +87,10 @@ func makeCmdMirrorEdit() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorEdit,
 		UsageLine: "edit <name>",
-		Short:     "edit properties of mirorr",
+		Short:     "edit mirror settings",
 		Long: `
-Command edit allows to change settings of mirror:
-filters.
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
 
 Example:
 
@@ -60,8 +99,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {
@@ -28,6 +29,8 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/mirror_rename.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -24,6 +25,11 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
 	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: mirror %s already exists", newName)

cmd/mirror_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -2,10 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {
@@ -28,20 +30,28 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	downloadSources := "no"
+	downloadSources := No
 	if repo.DownloadSources {
-		downloadSources = "yes"
+		downloadSources = Yes
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := No
+	if repo.DownloadUdebs {
+		downloadUdebs = Yes
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
-		filterWithDeps := "no"
+		filterWithDeps := No
 		if repo.FilterWithDeps {
-			filterWithDeps = "yes"
+			filterWithDeps = Yes
 		}
 		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
 	}
@@ -57,7 +67,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")

cmd/mirror_update.go

@@ -2,8 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -27,9 +32,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
 
-	verifier, err := getVerifier(context.flags)
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
+
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -39,21 +53,190 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	var filterQuery deb.PackageQuery
-
-	if repo.Filter != "" {
-		filterQuery, err = query.Parse(repo.Filter)
-		if err != nil {
-			return fmt.Errorf("unable to update: %s", err)
-		}
-	}
-
-	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch,
-		context.DependencyOptions(), filterQuery)
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(),
+		context.CollectionFactory().ChecksumCollection(), skipExistingPackages)
+
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	defer func() {
+		// on any interruption, unlock the mirror
+		err = context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.GoContextHandleSignals()
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true)
+
+	downloadQueue := make(chan int)
+
+	var (
+		errors  []string
+		errLock sync.Mutex
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
+	}
+
+	go func() {
+		for idx := range queue {
+			select {
+			case downloadQueue <- idx:
+			case <-context.Done():
+				return
+			}
+		}
+		close(downloadQueue)
+	}()
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case idx, ok := <-downloadQueue:
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						context,
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreMismatch,
+						maxTries)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					task.Done = true
+				case <-context.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	// Wait for all download goroutines to finish
+	wg.Wait()
+
+	context.Progress().ShutdownBar()
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false)
+
+	for idx := range queue {
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -80,9 +263,12 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	if len(args) == 1 {
+		q, err = query.Parse(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search [<package-query>]",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query.
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,143 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func printReferencesTo(p *deb.Package) (err error) {
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+
+	return err
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
+			for _, f := range p.Files() {
+				var path string
+				path, err = f.GetPoolPath(packagePool)
+				if err != nil {
+					return err
+				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matching query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -1,20 +1,21 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
-func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
-	if flags.Lookup("skip-signing").Value.Get().(bool) || context.Config().GpgDisableSign {
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
 
 	err := signer.Init()
 	if err != nil {
@@ -25,20 +26,6 @@ func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
 
 }
 
-func parsePrefix(param string) (storage, prefix string) {
-	i := strings.LastIndex(param, ":")
-	if i != -1 {
-		storage = param[:i]
-		prefix = param[i+1:]
-		if prefix == "" {
-			prefix = "."
-		}
-	} else {
-		prefix = param
-	}
-	return
-}
-
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",
@@ -50,6 +37,7 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishSnapshot(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
+			makeCmdPublishShow(),
 		},
 	}
 }

cmd/publish_drop.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,10 +21,12 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 		param = args[1]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+		context.CollectionFactory(), context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -47,5 +51,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
 	return cmd
 }

cmd/publish_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {
@@ -19,13 +20,13 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		if e != nil {
+			return e
 		}
 
 		if raw {
-			published = append(published, fmt.Sprintf("%s %s", repo.Prefix, repo.Distribution))
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
 		} else {
 			published = append(published, repo.String())
 		}
@@ -36,6 +37,8 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
+	context.CloseDatabase()
+
 	sort.Strings(published)
 
 	if raw {

cmd/publish_repo.go

@@ -37,9 +37,17 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_show.go

@@ -0,0 +1,81 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for component, sourceID := range repo.Sources {
+		var name string
+		if repo.SourceKind == deb.SourceSnapshot {
+			source, e := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == deb.SourceLocalRepo {
+			source, e := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,18 +2,19 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -27,14 +28,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	} else {
 		param = ""
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var (
 		sources = []interface{}{}
 		message string
 	)
 
-	if cmd.Name() == "snapshot" {
+	if cmd.Name() == "snapshot" { // nolint: goconst
 		var (
 			snapshot     *deb.Snapshot
 			emptyWarning = false
@@ -70,7 +71,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		if emptyWarning {
 			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
 		}
-	} else if cmd.Name() == "repo" {
+	} else if cmd.Name() == "repo" { // nolint: goconst
 		var (
 			localRepo    *deb.LocalRepo
 			emptyWarning = false
@@ -110,14 +111,35 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		panic("unknown command")
 	}
 
-	distribution := context.flags.Lookup("distribution").Value.String()
+	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
+	published.Label = context.Flags().Lookup("label").Value.String()
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
 
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
@@ -125,12 +147,18 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -150,14 +178,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
 		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
 			localStorage.PublicPath())
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
 	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
 		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
@@ -193,9 +221,17 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
-	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -2,16 +2,18 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
 
-	components := strings.Split(context.flags.Lookup("component").Value.String(), ",")
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
 	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
@@ -33,7 +35,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		names = args[1:]
 	}
 
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -42,7 +44,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "snapshot" {
+	if published.SourceKind != deb.SourceSnapshot {
 		return fmt.Errorf("unable to update: not a snapshot publish")
 	}
 
@@ -61,6 +63,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
 		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
@@ -74,12 +80,22 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.UpdateSnapshot(component, snapshot)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -89,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -106,7 +125,7 @@ func makeCmdPublishSwitch() *commander.Command {
 		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
-Command switches in-place published repository with new snapshot contents. All
+Command switches in-place published snapshots with new snapshot contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
@@ -114,19 +133,28 @@ For multiple component repositories, flag -component should be given with
 list of components to update. Corresponding snapshots should be given in the
 same order, e.g.:
 
-	aptly publish update -component=main,contrib wheezy wh-main wh-contrib
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
 
 Example:
 
-    $ aptly publish update wheezy ppa wheezy-7.5
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -20,7 +21,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	if len(args) == 2 {
 		param = args[1]
 	}
-	storage, prefix := parsePrefix(param)
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
@@ -29,7 +30,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "local" {
+	if published.SourceKind != deb.SourceLocalRepo {
 		return fmt.Errorf("unable to update: not a local repository publish")
 	}
 
@@ -43,12 +44,22 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.UpdateLocalRepo(component)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -58,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -92,7 +106,13 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/repo.go

@@ -20,6 +20,8 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -2,14 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
@@ -21,7 +20,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
 	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
@@ -40,131 +39,23 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	packageFiles := []string{}
-	failedFiles := []string{}
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	for _, location := range args[1:] {
-		info, err2 := os.Stat(location)
-		if err2 != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
-			failedFiles = append(failedFiles, location)
-			continue
-		}
-		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
-				if info.IsDir() {
-					return nil
-				}
+	var packageFiles, otherFiles, failedFiles []string
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
-					packageFiles = append(packageFiles, path)
-				}
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
-				return nil
-			})
-		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
-				packageFiles = append(packageFiles, location)
-			} else {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
-				failedFiles = append(failedFiles, location)
-				continue
-			}
-		}
+	var processedFiles, failedFiles2 []string
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		context.CollectionFactory().ChecksumCollection())
+	failedFiles = append(failedFiles, failedFiles2...)
+	if err != nil {
+		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
-	processedFiles := []string{}
-	sort.Strings(packageFiles)
-
-	for _, file := range packageFiles {
-		var (
-			stanza deb.Stanza
-			p      *deb.Package
-		)
-
-		candidateProcessedFiles := []string{}
-		isSourcePackage := strings.HasSuffix(file, ".dsc")
-
-		if isSourcePackage {
-			stanza, err = deb.GetControlFileFromDsc(file, verifier)
-
-			if err == nil {
-				stanza["Package"] = stanza["Source"]
-				delete(stanza, "Source")
-
-				p, err = deb.NewSourcePackageFromControlFile(stanza)
-			}
-		} else {
-			stanza, err = deb.GetControlFileFromDeb(file)
-			p = deb.NewPackageFromControlFile(stanza)
-		}
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		var checksums utils.ChecksumInfo
-		checksums, err = utils.ChecksumsForFile(file)
-		if err != nil {
-			return err
-		}
-
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
-		}
-
-		err = context.PackagePool().Import(file, checksums.MD5)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		candidateProcessedFiles = append(candidateProcessedFiles, file)
-
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
-				failedFiles = append(failedFiles, file)
-				break
-			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
-		}
-		if err != nil {
-			// some files haven't been imported
-			continue
-		}
-
-		err = context.CollectionFactory().PackageCollection().Update(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		err = list.Add(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
-		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
-		processedFiles = append(processedFiles, candidateProcessedFiles...)
-	}
+	processedFiles = append(processedFiles, otherFiles...)
 
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
@@ -173,11 +64,11 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if context.flags.Lookup("remove-files").Value.Get().(bool) {
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}
@@ -202,8 +93,8 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
 patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
 to the database. Files would be imported to internal package pool. For source packages, all required files are
 added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
@@ -216,6 +107,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_create.go

@@ -2,21 +2,46 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) != 1 {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
-	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
-	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
@@ -30,16 +55,21 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 func makeCmdRepoCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoCreate,
-		UsageLine: "create <name>",
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
 		Short:     "create local repository",
 		Long: `
 Create local package repository. Repository would be empty when
 created, packages could be added from files, copied or moved from
 another local repository or imported from the mirror.
 
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
 Example:
 
   $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
 `,
 		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
 	}
@@ -47,6 +77,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -34,7 +35,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: local repo is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
 

cmd/repo_edit.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/AlekSi/pointer"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,16 +26,30 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.flags.Lookup("comment").Value.String() != "" {
-		repo.Comment = context.flags.Lookup("comment").Value.String()
-	}
+	var uploadersFile *string
 
-	if context.flags.Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
-	}
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
 
-	if context.flags.Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -50,7 +67,7 @@ func makeCmdRepoEdit() *commander.Command {
 		UsageLine: "edit <name>",
 		Short:     "edit properties of local repository",
 		Long: `
-Command edit allows to change metadata of local repository:
+Command edit allows one to change metadata of local repository:
 comment, default distribution and component.
 
 Example:
@@ -63,6 +80,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = context.GetVerifier()
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, failedFiles2 []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplateString,
+		context.Progress(), context.CollectionFactory().LocalRepoCollection(), context.CollectionFactory().PackageCollection(),
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(),
+		uploaders, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {
@@ -22,9 +23,9 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-			if err != nil {
-				return err
+			e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
@@ -33,6 +34,8 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {

cmd/repo_move.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
@@ -33,7 +34,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		srcRepo    *deb.LocalRepo
 	)
 
-	if command == "copy" || command == "move" {
+	if command == "copy" || command == "move" { // nolint: goconst
 		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
@@ -49,7 +50,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 
 		srcRefList = srcRepo.RefList()
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
 		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
@@ -87,7 +88,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
@@ -114,18 +115,18 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var verb string
 
-	if command == "move" {
+	if command == "move" { // nolint: goconst
 		verb = "moved"
-	} else if command == "copy" {
+	} else if command == "copy" { // nolint: goconst
 		verb = "copied"
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		verb = "imported"
 	}
 
@@ -135,7 +136,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcList.Remove(p)
 		}
 		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
@@ -145,7 +146,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
@@ -155,7 +156,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
 			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)

cmd/repo_remove.go

@@ -2,8 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -54,7 +55,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))

cmd/repo_rename.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/repo_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/repo_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -29,9 +30,12 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(repo.RefList())
 	}

cmd/run.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*ctx.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			ctx.Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,16 +2,17 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
@@ -22,12 +23,23 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
 	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
 
-	listen := context.flags.Lookup("listen").Value.String()
+	listen := context.Flags().Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 
@@ -48,9 +60,9 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		if e != nil {
+			return e
 		}
 
 		sources = append(sources, repo.String())
@@ -78,13 +90,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
 			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
-		if utils.StrSliceHasItem(repo.Architectures, "source") {
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
 				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot.go

@@ -18,6 +18,8 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
 			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
 	}
 }

cmd/snapshot_create.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -12,7 +13,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
@@ -23,6 +24,11 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
 		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
@@ -32,7 +38,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
-	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
 		// aptly snapshot create snap from repo repo
 		var repo *deb.LocalRepo
 

cmd/snapshot_diff.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -13,7 +14,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
 
 	// Load <name-a> snapshot
 	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])

cmd/snapshot_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -35,7 +36,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {

cmd/snapshot_filter.go

@@ -0,0 +1,108 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+
+	// Load <source> snapshot
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -2,51 +2,11 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
-// Snapshot sorting methods
-const (
-	SortName = iota
-	SortTime
-)
-
-type snapshotListToSort struct {
-	list       []*deb.Snapshot
-	sortMethod int
-}
-
-func parseSortMethod(sortMethod string) (int, error) {
-	switch sortMethod {
-	case "time", "Time":
-		return SortTime, nil
-	case "name", "Name":
-		return SortName, nil
-	}
-
-	return -1, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
-}
-
-func (s snapshotListToSort) Swap(i, j int) {
-	s.list[i], s.list[j] = s.list[j], s.list[i]
-}
-
-func (s snapshotListToSort) Less(i, j int) bool {
-	switch s.sortMethod {
-	case SortName:
-		return s.list[i].Name < s.list[j].Name
-	case SortTime:
-		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
-	}
-	panic("unknown sort method")
-}
-
-func (s snapshotListToSort) Len() int {
-	return len(s.list)
-}
-
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 0 {
@@ -57,33 +17,24 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshotsToSort := &snapshotListToSort{}
-	snapshotsToSort.list = make([]*deb.Snapshot, context.CollectionFactory().SnapshotCollection().Len())
-	snapshotsToSort.sortMethod, err = parseSortMethod(sortMethodString)
-	if err != nil {
-		return err
-	}
-
-	i := 0
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		snapshotsToSort.list[i] = snapshot
-		i++
-
-		return nil
-	})
-
-	sort.Sort(snapshotsToSort)
+	collection := context.CollectionFactory().SnapshotCollection()
 
 	if raw {
-		for _, snapshot := range snapshotsToSort.list {
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
-		}
+			return nil
+		})
 	} else {
-		if len(snapshotsToSort.list) > 0 {
+		if collection.Len() > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshotsToSort.list {
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -91,8 +42,8 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
 		}
 	}
-	return err
 
+	return err
 }
 
 func makeCmdSnapshotList() *commander.Command {

cmd/snapshot_merge.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
@@ -28,22 +29,22 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	latest := context.flags.Lookup("latest").Value.Get().(bool)
-    noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 
-    if noRemove && latest {
-        return fmt.Errorf("-no-remove and -latest can't be specified together")
-    }
+	if noRemove && latest {
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
+	}
 
 	overrideMatching := !latest && !noRemove
 
 	result := sources[0].RefList()
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {
-		deb.FilterLatestRefs(result)
+		result.FilterLatestRefs()
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -84,7 +85,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
-    cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"sort"
 	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotPull(cmd *commander.Command, args []string) error {
@@ -17,9 +18,9 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
-	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
-	allMatches := context.flags.Lookup("all-matches").Value.Get().(bool)
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
 
 	// Load <name> snapshot
 	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
@@ -91,11 +92,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 		// Add architecture filter
-		queries[i] = &deb.AndQuery{queries[i], archQuery}
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -129,7 +130,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	})
 	alreadySeen = nil
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot

cmd/snapshot_rename.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/snapshot_search.go

@@ -0,0 +1,142 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" { // nolint: goconst
+		var snapshot *deb.Snapshot
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		var repo *deb.RemoteRepo
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" { // nolint: goconst
+		var repo *deb.LocalRepo
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	if len(args) == 2 {
+		q, err = query.Parse(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -29,8 +31,40 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+	if len(snapshot.SourceIDs) > 0 {
+		fmt.Printf("Sources:\n")
+		for _, sourceID := range snapshot.SourceIDs {
+			var name string
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+			if name != "" {
+				fmt.Printf("  %s [%s]\n", name, snapshot.SourceKind)
+			}
+		}
+	}
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		ListPackagesRefList(snapshot.RefList())
 	}

cmd/snapshot_verify.go

@@ -2,9 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
@@ -31,25 +32,25 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
 		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s", filename)
+		}
+
+		fmt.Print("Reading file...\n\n")
+
+		var file *os.File
+		file, err = os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			err = context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+	  $ aptly task run
+	  > repo create local
+	  > repo add local pkg1
+	  > publish repo local
+	  > serve
+	  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}

cmd/version.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
+
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/smira/commander"
 )
 

completion.d/README-zsh.md

@@ -0,0 +1,65 @@
+# aptly-zsh
+zsh completion for aptly
+
+The zsh completion function and this README were imported from
+[steinymity's repository](https://github.com/steinymity/aptly-zsh).
+
+[Aptly](https://www.aptly.info/) is a great tool to setup Debian APT repositories
+and mirrors. However, it's sometimes hard to remember all the command line
+syntax and names of all options. Therefore I wrote this zsh completion modules
+for aptly.
+
+## License
+
+This project is licensed under the terms of the MIT license. See file `LICENSE`
+for details.
+
+## Installation
+
+Clone/copy the file `_aptly` to a place in your `$fpath` (show with
+`echo $fpath`), or create a new directory and extend the fpath:
+
+    mkdir -p ~/.zsh/functions
+    fpath=(~/.zsh/functions $fpath)
+    editor ~/.zsh/functions/_aptly
+
+To profit most from the provided help messages and completions, make sure that
+your zsh is setup properly. I have tested with the grml-zsh configuration that
+is [available on Github](https://github.com/grml/grml-etc-core/) and on the
+[grml homepage](http://grml.org/zsh/).
+
+## Compatibility
+
+The command line completion was developed based on the manpage of aptly 1.2.0
+(currently in Debian Testing). However, most completions will work on older
+versions (e.g., 0.9.7 in Debian Stable), too.
+
+The completion function completes most arguments and options that can be passed
+to aptly, including mirror/repository/snapshot/publish names. However, not all
+arguments are handled yet. See the next section for known limitations.
+
+## Known Bugs and Limitation
+
+ * Boolean options are always completed with an explicit value `true` or
+   `false`, although omitting the value is implicitly interpreted as `true`.
+ * The source and destination names of copy and move operations must not be the
+   same. This is currently not enforced.
+ * The package query and display format strings are currently not completed.
+ * Endpoints are not completed.
+ * In `publish snapshot` there is no connection between the number of
+   components passed to `-component` and the number of given snapshots to
+   publish. Furthermore, the help text for `endpoint:prefix` disappears
+   after its first possible location.
+ * In `publish switch` the distribution can be set independently of the
+   `endpoint:prefix` (i.e., all published distributions can be combined with
+   all published `endpoint:prefix`).
+ * Neither `publish switch` nor `publish update` check if publish was created
+   from a snapshot or directly from a local repo.
+ * Task commands are not completed. There are no commas added.
+ * `help` won't complete after a sub command (i.e., `aptly mirror list help`).
+ * Probably more, feel free to open issues, and submit patches/merge requests.
+
+--------------------------------------------------------------------------------
+
+Maximilian Stein <m@steiny.biz>
+2018-02-03

completion.d/_aptly

@@ -0,0 +1,617 @@
+#compdef aptly
+################################################################################
+# Copyright (C) 2018 Maximilian Stein <m@steiny.biz>
+#   Acknowledgement: Many texts where copied from aptly(1)
+#
+# This project is licensed under the terms of the MIT license.
+# See file LICENSE for details.
+################################################################################
+# zsh completion script for Aptly (http://aptly.info/)
+################################################################################
+
+# see https://man.cx/aptly
+
+local curcontext="$curcontext" state line cmd subcmd ret=1
+
+local arch_list=(amd64 arm64 armel armhf i386 mips mipsel mips64el ppc64el s390x)
+local bool="bool:{_wanted -V values expl 'bool' compadd true false}"
+local dists=({wheezy,jessie,stretch}{,-updates,-backports,-backports-sloopy} buster sid experimental)
+local components=(main contrib non-free)
+local aptly_query="aptly package query: "
+local aptly_format="aptly package display format: "
+local aptly_uploaders="-uploaders-file=[uploaders.json to be used when including .changes into this repository]:uploaders file:_files -g '*.json'"
+local keyring="*-keyring=[gpg keyring to use when verifying Release file (could be specified multiple times)]:keyring file:_files -g '*.gpg'"
+
+# complete command
+(( $+functions[_aptly-cmd] )) ||
+    _aptly-cmd() {
+        _values "aptly command category" \
+            "mirror[manage, update mirrors of remote repositories]" \
+            "repo[manage local package repositories, add, remove, move, copy packages]" \
+            "snapshot[create, merge, manage snapshots]" \
+            "package[perform operation on the whole collection of packages]" \
+            "publish[publish snapshot or local repository]" \
+            "db[cleanup database and package pool, recover database after failure]" \
+            "task[multi-command tasks]" \
+            "serve[quickly serve published repositories via HTTP]" \
+            "config[configuration management]" \
+            "graph[generate dependency graph]" \
+            "api[REST API service]"
+        ret=0
+}
+
+# complete subcommand
+(( $+functions[_aptly-subcmd] )) ||
+    _aptly-subcmd() {
+        local cmd=$1
+        case $cmd in
+            mirror)
+                _values "mirror commands" \
+                    "create[create new mirror of remote repository]" \
+                    "list[show full list of mirrors]" \
+                    "show[show details about a mirror]" \
+                    "drop[delete a mirror]" \
+                    "update[update a mirror]" \
+                    "rename[change name of a mirror]" \
+                    "edit[change settings of a mirror]" \
+                    "search[search mirror for packages matching query]"
+                ret=0 ;;
+            repo)
+                _values "repo commands" \
+                    "add[add packages to local repository]" \
+                    "copy[copy packages between local repositories]" \
+                    "create[create local repository]" \
+                    "drop[delete local repository]" \
+                    "edit[edit properties of local repository]" \
+                    "import[import packages from mirror to local repository]" \
+                    "list[list local repositories]" \
+                    "move[move packages between local repositories]" \
+                    "remove[remove packages from local repository]" \
+                    "show[show details about local repository]" \
+                    "rename[renames local repository]" \
+                    "search[search repo for packages matching query]" \
+                    "include[add packages to local repositories based on .changes files]"
+                ret=0 ;;
+            snapshot)
+                _values "snapshot commands" \
+                    "create[create snapshot of mirror or local repository]" \
+                    "list[list snapshots]" \
+                    "show[show details about snapshot]" \
+                    "verify[verify dependencies in snapshot]" \
+                    "pull[pull packages from another snapshot]" \
+                    "diff[show difference between two snapshots]" \
+                    "merge[merge snapshots]" \
+                    "drop[delete snapshot]" \
+                    "rename[rename snapshot]" \
+                    "search[search snapshot for packages matching query]" \
+                    "filter[filter packages in snapshot producing another snapshot]"
+                ret=0 ;;
+            publish)
+                _values "publish commands" \
+                    "drop[remove published repository]" \
+                    "list[list published repositories]" \
+                    "repo[publish local repository]" \
+                    "snapshot[publish snapshot]" \
+                    "switch[update published repository by switching to new snapshot]" \
+                    "update[update published local repository]" \
+                    "show[shows details of published repository]"
+                ret=0 ;;
+            package)
+                _values "package commands" \
+                    "search[search for packages matching query]" \
+                    "show[show details about packages matching query]"
+                ret=0 ;;
+            db)
+                _values "db commands" \
+                    "cleanup[cleanup db and package pool]" \
+                    "recover[recover db after crash]"
+                ret=0 ;;
+            serve)
+                # no subcommand here
+                _arguments '1:: :' \
+                    '-listen=[host:port for HTTP listening]:host\:port: '
+                ret=0 ;;
+            api)
+                _values "api commands" \
+                    "serve[start api http service]"
+                ret=0 ;;
+            graph)
+                # no subcommand here
+                _arguments '*:' \
+                    '-format=[render graph to specified format]:image format:(png svg pdf)' \
+                    '-layout=[create a more vertical or more horizontal graph layout]:layout:(horizontal vertical)' \
+                    '-output=[specify output filename, default is to open result in viewer]:output file:_files'
+                ret=0 ;;
+            config)
+                _values "config commands" \
+                    "show[show current aptly config]"
+                ret=0 ;;
+            task)
+                _values "task commands" \
+                    "run[run aptly tasks]"
+                ret=0 ;;
+        esac
+}
+
+# complete parameters
+(( $+functions[_aptly-param] )) ||
+    _aptly-param() {
+        local cmd=$1 subcmd=$2
+
+        local config=$opt_args[-config]
+        [[ -n $config ]] && config="-config=$config"
+
+        # get list of mirrors, or ' ' if none
+        get_mirrors() {
+            # retrieve list of mirrors
+            local mirrors=($(aptly $config mirror list -raw=true 2>/dev/null))
+            # a single space causes just the help text to be shown
+            [[ -z $mirrors ]] && mirrors=" " || mirrors="($mirrors)"
+            echo $mirrors
+        }
+        # get lists of repos or ' ' if none
+        get_repos() {
+            # retrieve repo list
+            local repos=($(aptly $config repo list -raw=true 2>/dev/null))
+            [[ -z $repos ]] && repos=" " || repos="($repos)"
+            echo $repos
+        }
+        # get list of snapshots or ' ' if none
+        get_snapshots() {
+            local snapshots=($(aptly $config snapshot list -raw=true 2>/dev/null))
+            [[ -z $snapshots ]] && snapshots=" " || snapshots="($snapshots)"
+            echo $snapshots
+        }
+        # get list of gpg keys or ' ' if none
+        get_gpg_key_ids() {
+            local gpg_keys=($(gpg --quiet --batch  --keyid-format long --list-secret-keys --with-colons 2>/dev/null | grep '^sec' | cut -d ':' -f 5))
+            [[ -z $gpg_keys ]] && gpg_keys=" " || gpg_keys="($gpg_keys)"
+            echo $gpg_keys
+        }
+
+        ret=0
+        case $cmd in
+            mirror)
+                local mirrors=$(get_mirrors)
+
+                case $subcmd in
+                    create)
+                        _arguments \
+                            "-filter=[filter packages in mirror]:$aptly_query" \
+                            "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
+                            "-force-architecture=[(only with architecture list) skip check that requested architectures are listed in Release file]:$bool" \
+                            "-force-components=[(only with component list) skip check that requested components are listed in Release file]:$bool" \
+                            "-ignore-signatures=[disable verification of Release file signatures]:$bool" \
+                            $keyring \
+                            "-with-sources=[download source packages in addition to binary packages]:$bool" \
+                            "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine-readable format]:$bool"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show detailed list of packages and versions stored in the mirror]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[force mirror deletion even if used by snapshots]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    update)
+                        _arguments \
+                            "-download-limit=[limit download speed (kB/s)]:kB/s: " \
+                            "-force=[force update mirror even if it is locked by another process]:$bool" \
+                            "-ignore-checksums=[ignore checksum mismatches while downloading package files and metadata]:$bool" \
+                            "-ignore-signatures=[disable verification of Release file signatures]:$bool" \
+                            $keyring \
+                            "-max-tries=[max download tries till process fails with download error]:number: " \
+                            "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    rename)
+                        _arguments \
+                            "2:old mirror name:$mirrors" ":new mirror name: "
+                        ;;
+                    edit)
+                        _arguments \
+                            "-filter=[filter packages in mirror]:$aptly_query" \
+                            "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
+                            "-with-sources=[download source packages in addition to binary packages]:$bool" \
+                            "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:mirror name:$mirrors" ":$aptly_query"
+                        ;;
+                esac
+                ;;
+
+            repo)
+
+
+                local repos=$(get_repos)
+
+                local create_edit=("-comment=[any text that would be used to described local repository]:comment: "
+                            "-component=[default component when publishing]:component:($components)"
+                            "-distribution=[default distribution when publishing]:distribution:($dists)"
+                            $aptly_uploaders
+                            )
+
+                case $subcmd in
+                    add)
+                        _arguments \
+                            "-force-replace=[when adding package that conflicts with existing package, remove existing package]:$bool" \
+                            "-remove-files=[remove files that have been imported successfully into repository]:$bool" \
+                            "(-)2:repo name:$repos" "*:package files:_files -g '*.{udeb,deb,dsc}'"
+                        ;;
+                    copy)
+                        _arguments \
+                            "-dry-run=[don’t copy, just show what would be copied]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$$bool" \
+                            "(-)2:src repo name:$repos" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    create)
+                        local snapshots=$(get_snapshots)
+
+                        _arguments \
+                            ${create_edit[@]} \
+                            "(-)2:new repo name: " \
+                            "3:::('from')" "4:::('snapshot')" "5::snapshot:$snapshots"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[force local repo deletion even if used by snapshots]:$bool" \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    edit)
+                        _arguments \
+                            ${create_edit[@]} \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    import)
+                        local mirrors=$(get_mirrors)
+
+                        _arguments \
+                            "-dry-run=[don’t import, just show what would be imported]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$bool" \
+                            "(-)2:src mirror name:$mirrors" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine−readable format]:$bool"
+                        ;;
+                    move)
+                        _arguments \
+                            "-dry-run=[don’t move, just show what would be moved]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$bool" \
+                            "(-)2:srv repo name:$repos" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    remove)
+                        _arguments \
+                            "-dry-run=[don’t remove, just show what would be removed]:$bool" \
+                            "(-)2:repo name:$repos" "*:$aptly_query"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show list of packages]:$bool" \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    rename)
+                        _arguments \
+                            "2:old repo name:$repos" ":new repo name: "
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:repo name:$repos" ":$aptly_query"
+                        ;;
+                    include)
+                        _arguments '1:: :' \
+                            "-accept-unsigned=[accept unsigned .changes files]:$bool" \
+                            "-force-replace=[when adding package that conflicts with existing package, remove existing package]:$bool" \
+                            "-ignore-signatures=[disable verification of .changes file signature]:$bool" \
+                            $keyring \
+                            "-no-remove-files=[don’t remove files that have been imported successfully into repository]:$bool" \
+                            "-repo=[which repo should files go to, defaults to Distribution field of .changes file]:repo name:$repos" \
+                            $aptly_uploaders \
+                            "(-)*:changes files/directories:_files -g '*.changes'"
+                        ;;
+                esac
+                ;;
+            snapshot)
+                local snapshots=$(get_snapshots)
+
+                case $subcmd in
+                    create)
+                        local mirrors=$(get_mirrors)
+                        local repos=$(get_repos)
+
+                        _arguments -C \
+                            '(-)2:new snapshot name: ' \
+                            '3: :->src1' \
+                            '4:: :->src2' '5:: :->src3'
+
+                        case $state in
+                            src1)
+                                _values 'snapshot src' 'from' 'empty' ;;
+                            src2)
+                                if [[ $line[3] == from ]]; then
+                                    _values 'snapshot src' 'mirror' 'repo'
+                                fi
+                                ;;
+                            src3)
+                                if [[ $line[3] == from ]]; then
+                                    case $line[4] in
+                                        mirror)
+                                            _arguments "5:mirror name:$mirrors" ;;
+                                        repo)
+                                            _arguments "5:repo name:$repos" ;;
+                                    esac
+                                fi
+                                ;;
+                        esac
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine−readable format]:$bool" \
+                            "-sort=[display list in ’name’ or creation ’time’ order]:sort order:((name\:'alphabetical order' time\:'chronological order'))"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show list of packages]:$bool" \
+                            "(-)2:snapshot name:$snapshots"
+                        ;;
+                    verify)
+                        _arguments '1:: :' \
+                            "(-)2:snapshot name:$snapshots" "*::more snapshots:$snapshots"
+                        ;;
+                    pull)
+                        _arguments \
+                            "-all-matches=[pull all the packages that satisfy the dependency version requirements]:$bool" \
+                            "-dry-run=[don’t create destination snapshot, just show what would be pulled]:$bool" \
+                            "-no-deps=[don’t process dependencies, just pull listed packages]:$bool" \
+                            "-no-remove=[don’t remove other package versions when pulling package]:$bool" \
+                            "(-)2:to snapshot name:$snapshots" "3:src snapshot name:$snapshots" "4:new dest snapshot name: " \
+                            "*:$aptly_query"
+                        ;;
+                    diff)
+                        _arguments \
+                            "-only-matching=[display diff only for matching packages (don’t display missing packages)]:$bool" \
+                            "(-)2:snapshot name a:$snapshots" "3:snapshot name b:$snapshots"
+                        ;;
+                    merge)
+                        _arguments \
+                            "-latest=[use only the latest version of each package]:$bool" \
+                            "-no-remove=[don’t remove duplicate arch/name packages]:$bool" \
+                            "(-)2:new dest snapshot name: " "*:source snapshot name(s):$snapshots"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[remove snapshot even if it was used as source for other snapshots]:$bool" \
+                            "(-)2:snapshot name:$snapshots"
+                        ;;
+                    rename)
+                        _arguments '1:: :' \
+                            "2:old snapshot name:$snapshots" "3:new snapshot name: "
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:snapshot name:$snapshots" ":$aptly_query"
+                        ;;
+                    filter)
+                        _arguments \
+                            "-with-deps=[include dependent packages as well]:$bool" \
+                            "(-)2:src snapshot name:$snapshots" "3:new dest snapshot name: " "*:$aptly_query"
+                        ;;
+                esac
+                ;;
+            publish)
+                # read lines of output into
+                # format of each item: <ep:prefix> <distribution>
+                local -a publish_prefixes
+                local -a publish_dists
+                for line in ${(@f)"$(aptly $config publish list -raw=true 2>/dev/null)"}
+                {
+                    publish_prefixes+=($line[(ws: :)1])
+                    publish_dists+=($line[(ws: :)2])
+                }
+                publish_prefixes_uniq=(${(@u)publish_prefixes})
+                publish_dists_uniq=(${(@u)publish_dists})
+                [[ -z $publish_prefixes_uniq ]] && publish_prefixes_uniq=" " || publish_prefixes_uniq="($publish_prefixes_uniq)"
+                [[ -z $publish_dists_uniq ]]    && publish_dists_uniq=" "    || publish_dists_uniq="($publish_dists_uniq)"
+
+                local gpg_keys=$(get_gpg_key_ids)
+
+                # common options for publishing
+                # TODO: is the keyring parameter correct?
+                local publish_update_options=(
+                            "-batch=[run GPG with detached tty]:$bool"
+                            "-force-overwrite=[overwrite files in package pool in case of mismatch]:$bool"
+                            "-gpg-key=[GPG key ID to use when signing the release]:gpg key id:$gpg_keys"
+                            "-keyring=[GPG keyring to use (instead of default)]:keyring file:_files -g '*.gpg'"
+                            "-passphrase=[GPG passhprase for the key (warning: could be insecure)]:passphrase: "
+                            "-passphrase-file=[GPG passhprase−file for the key (warning: could be insecure)]:passphrase file:_files"
+                            "-secret-keyring=[GPG secret keyring to use (instead of default)]:secret-keyring:_files"
+                            "-skip-contents=[don’t generate Contents indexes]:$bool"
+                            "-skip-signing=[don’t sign Release files with GPG]:$bool"
+                )
+                local components_options=(
+                            "-component=[component name to publish (for multi−component publishing, separate components with commas)]:components:_values -s , components $components"
+                )
+                local publish_options=(
+                            "-butautomaticupgrades=[set value for ButAutomaticUpgrades field]:$bool"
+                            "-distribution=[distribution name to publish]:distribution:($dists)"
+                            "-label=[label to publish]:label: "
+                            "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
+                            "-origin=[origin name to publish]:origin: "
+                            ${components_options[@]}
+                )
+
+                local endpoint_prefix="[endpoint\:]prefix"
+
+                case $subcmd in
+                    repo)
+                        local repos=$(get_repos)
+                        _arguments \
+                            ${publish_options[@]} \
+                            ${publish_update_options[@]} \
+                            "(-)2:repo name:$repos" "3::$endpoint_prefix: "
+                        ;;
+                    snapshot)
+                        local snapshots=$(get_snapshots)
+                        _arguments '1:: :' \
+                            ${publish_options[@]} \
+                            ${publish_update_options[@]} \
+                            "(-)*:snapshot name:$snapshots" "3::$endpoint_prefix: "
+                        ;;
+                    switch)
+                        local snapshots=$(get_snapshots)
+                        _arguments \
+                            ${publish_update_options[@]} \
+                            ${components_options[@]} \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq" \
+                            "*:new snapshot name:$snapshots"
+                        ;;
+                    update)
+                        _arguments \
+                            ${publish_update_options[@]} \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq"
+                        ;;
+                    show)
+                        _arguments '1:: :' \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq"
+                        ;;
+                esac
+                ;;
+            package)
+                case $subcmd in
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "(-)2:$aptly_query"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-files=[display information about files from package pool]:$bool" \
+                            "-with-references=[display information about mirrors, snapshots and local repos referencing this package]:$bool" \
+                            "(-)2:$aptly_query"
+                        ;;
+                esac
+                ;;
+            db)
+                case $subcmd in
+                    cleanup)
+                        _arguments '1:: :' \
+                            "-dry-run=[don’t delete anything]:$bool" \
+                            "-verbose=[be verbose when loading objects/removing them]:$bool"
+                        ;;
+                    recover)
+                        # nothing to complete...
+                        ;;
+                esac
+                ;;
+            serve)
+                # completed in _aptly-subcmd
+                ;;
+            api)
+                case $subcmd in
+                    serve)
+                        _arguments '1:: :' \
+                            "-listen=[host:port for HTTP listening or unix://path to listen on a Unix domain socket]:host\:port or unix\://path: " \
+                            "-no-lock=[don’t lock the database]:$bool"
+                        ;;
+                esac
+                ;;
+            graph)
+                # completed in _aptly-subcmd
+                ;;
+            config)
+                case $subcmd in
+                    show)
+                        # nothing to do
+                        ;;
+                esac
+                ;;
+            task)
+                case $subcmd in
+                    run)
+                        _arguments '1:: :' \
+                            "(2)-filename=[specifies the filename that contains the commands to run]:filename:_files" \
+                            "(-filename)*::comma-separated command list: "
+                esac
+                ;;
+        esac
+}
+
+# main completion
+_arguments -C \
+    "-architectures=[list of architectures to consider (comma−separated), default to all available]:architectures:_values -s , architectures $arch_list" \
+    "-config=[location of configuration file]:file:_files -g '*.conf'" \
+    "-db-open-attempts=[number of attempts to open DB if it’s locked by other instance]:number:()" \
+    "-dep-follow-all-variants=[when processing dependencies, follow a & b if dependency is ’a|b’]:$bool" \
+    "-dep-follow-recommends=[when processing dependencies, follow Recommends]:$bool" \
+    "-dep-follow-source=[when processing dependencies, follow from binary to Source packages]:$bool" \
+    "-dep-follow-suggests=[when processing dependencies, follow Suggests]:$bool" \
+    "-dep-verbose-resolve=[when processing dependencies, print detailed logs]:$bool" \
+    "-gpg-provider=[PGP implementation]:gpg provider:((gpg\:'external gpg' internal\:'Go internal implementation'))" \
+    '(-)1: :->cmds' \
+    '2: :->subcmd' \
+    '*:: :->args' && ret=0
+
+cmd=$line[1]
+subcmd=$line[2]
+
+case $state in
+    cmds)
+        _aptly-cmd
+        _arguments '(-)1:: :((help\:integrated\ command\ help))'
+        ;;
+    subcmd)
+        case $cmd in
+            help)
+                _aptly-cmd
+                ;;
+            *)
+                _aptly-subcmd $cmd
+                _arguments '(-)2:: :((help\:integrated\ command\ help))'
+                ;;
+        esac
+        ;;
+
+    args)
+        # help anywhere in line?
+        if [[ ${line[(i)help]} -le ${#line} ]]; then
+            if [[ ${#line} -le 3 ]]; then
+                if [[ $line[1] == help ]]; then
+                    _aptly-subcmd $subcmd
+                elif [[ $line[2] == help ]]; then
+                    _aptly-subcmd $cmd
+                fi
+            fi
+        else
+            _aptly-param $cmd $subcmd
+            # this somehow destroys parameter completion, so disable it for now
+            #_arguments '(-)3:: :((help\:integrated\ command\ help))'
+        fi
+        ret=0
+        ;;
+esac
+
+return ret
+
+# mode: Shell-Script
+# sh-indentation: 4
+# indent-tabs-mode: nil
+# sh-basic-offset: 4
+# End:
+# vim: ft=zsh sw=4 ts=4 et

completion.d/aptly

@@ -0,0 +1,633 @@
+#!/bin/bash
+
+# (The MIT License)
+#
+# Copyright (c) 2014 Andrey Smirnov
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+__aptly_mirror_list()
+{
+  aptly mirror list -raw
+}
+
+__aptly_repo_list()
+{
+  aptly repo list -raw
+}
+
+__aptly_snapshot_list()
+{
+  aptly snapshot list -raw
+}
+
+__aptly_published_distributions()
+{
+  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
+}
+
+__aptly_published_prefixes()
+{
+  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
+}
+
+__aptly_prefixes_for_distribution()
+{
+  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
+}
+
+_aptly()
+{
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+
+    commands="api config db graph mirror package publish repo serve snapshot task version"
+    options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
+    db_subcommands="cleanup recover"
+    mirror_subcommands="create drop edit show list rename search update"
+    publish_subcommands="drop list repo snapshot switch update"
+    snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
+    repo_subcommands="add copy create drop edit import include list move remove rename search show"
+    package_subcommands="search show"
+    task_subcommands="run"
+    config_subcommands="show"
+    api_subcommands="serve"
+
+    local cmd subcmd numargs numoptions i
+
+    numargs=0
+    numoptions=0
+
+    for (( i=1; i < $COMP_CWORD; i++ )); do
+      if [[ -n "$cmd" ]]; then
+        if [[ ! -n "$subcmd" ]]; then
+          subcmd=${COMP_WORDS[i]}
+          numargs=$(( COMP_CWORD - i - 1 ))
+        else
+          if [[ "${COMP_WORDS[i]}" == -* ]]; then
+            numoptions=$(( numoptions + 1 ))
+            numargs=$(( numargs - 1 ))
+          fi
+        fi
+      else
+        if [[ ! "${COMP_WORDS[i]}" == -* ]]; then
+          cmd=${COMP_WORDS[i]}
+        fi
+      fi
+    done
+
+    if [[ ! -n "$cmd" ]];
+    then
+      case "$cur" in
+            -*)
+              COMPREPLY=($(compgen -W "${options}" -- ${cur}))
+              return 0
+            ;;
+            *)
+              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
+              return 0
+            ;;
+        esac
+    fi
+
+    if [[ ! -n "$subcmd" ]];
+    then
+        case "$prev" in
+            "db")
+              COMPREPLY=($(compgen -W "${db_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "mirror")
+              COMPREPLY=($(compgen -W "${mirror_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "repo")
+              COMPREPLY=($(compgen -W "${repo_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "snapshot")
+              COMPREPLY=($(compgen -W "${snapshot_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "publish")
+              COMPREPLY=($(compgen -W "${publish_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "package")
+              COMPREPLY=($(compgen -W "${package_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "task")
+              COMPREPLY=($(compgen -W "${task_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "config")
+              COMPREPLY=($(compgen -W "${config_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "api")
+              COMPREPLY=($(compgen -W "${api_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            *)
+            ;;
+        esac
+    fi
+
+    case "$cmd" in
+      "mirror")
+        case "$subcmd" in
+          "create")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                return 0
+              fi
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "repo")
+        case "$subcmd" in
+          "add")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-force-replace -remove-files" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                _filedir '@(deb|dsc|udeb)'
+                return 0
+              ;;
+            esac
+          ;;
+          "copy"|"move")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "create")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "from" -- ${cur}))
+                return 0
+              ;;
+              2)
+                COMPREPLY=($(compgen -W "snapshot" -- ${cur}))
+                return 0
+              ;;
+              3)
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "include")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
+                else
+                  comptopt -o filenames 2>/dev/null
+                  COMPREPLY=($(compgen -f -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+            esac
+          ;;
+          "import")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "remove")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "snapshot")
+        case "$subcmd" in
+          "create")
+            case $numargs in
+              1)
+                COMPREPLY=($(compgen -W "from empty" -- ${cur}))
+                return 0
+              ;;
+              2)
+                if [[ "$prev" == "from" ]]; then
+                  COMPREPLY=($(compgen -W "mirror repo" -- ${cur}))
+                  return 0
+                fi
+              ;;
+              3)
+                if [[ "$prev" == "mirror" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                  return 0
+                fi
+                if [[ "$prev" == "repo" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                  return 0
+                fi
+              ;;
+            esac
+          ;;
+          "diff")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-only-matching" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw -sort=" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "merge")
+            if [[ $numargs -gt 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-latest" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "pull")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-all-matches -dry-run -no-deps -no-remove" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "filter")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "verify")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "publish")
+        case "$subcmd" in
+          "snapshot"|"repo")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+              else
+                if [[ "$subcmd" == "snapshot" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_published_prefixes)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "switch")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -ge 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "package")
+        case "$subcmd" in
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-files -with-references" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "serve")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "graph")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-format= -output=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "api")
+        case "$subcmd" in
+          "serve")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "db")
+        case "$subcmd" in
+          "cleanup")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+    esac
+} && complete -F _aptly aptly

console/progress.go

@@ -2,18 +2,23 @@ package console
 
 import (
 	"fmt"
-	"github.com/cheggaaa/pb"
-	"github.com/smira/aptly/aptly"
-	"github.com/wsxiaoys/terminal/color"
+	"os"
 	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/cheggaaa/pb"
+	"github.com/wsxiaoys/terminal/color"
 )
 
 const (
 	codePrint = iota
+	codePrintStdErr
 	codeProgress
 	codeHideProgress
 	codeStop
 	codeFlush
+	codeBarEnabled
+	codeBarDisabled
 )
 
 type printTask struct {
@@ -25,7 +30,6 @@ type printTask struct {
 // Progress is a progress displaying subroutine, it allows to show download and other operations progress
 // mixed with progress bar
 type Progress struct {
-	stop     chan bool
 	stopped  chan bool
 	queue    chan printTask
 	bar      *pb.ProgressBar
@@ -81,6 +85,8 @@ func (p *Progress) InitBar(count int64, isBytes bool) {
 			p.bar.SetUnits(pb.U_BYTES)
 			p.bar.ShowSpeed = true
 		}
+
+		p.queue <- printTask{code: codeBarEnabled}
 		p.bar.Start()
 	}
 }
@@ -91,6 +97,7 @@ func (p *Progress) ShutdownBar() {
 		return
 	}
 	p.bar.Finish()
+	p.queue <- printTask{code: codeBarDisabled}
 	p.bar = nil
 	p.queue <- printTask{code: codeHideProgress}
 }
@@ -122,25 +129,41 @@ func (p *Progress) Printf(msg string, a ...interface{}) {
 	p.queue <- printTask{code: codePrint, message: fmt.Sprintf(msg, a...)}
 }
 
+// PrintfStdErr does printf but in safe manner to stderr
+func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
+	p.queue <- printTask{code: codePrintStdErr, message: fmt.Sprintf(msg, a...)}
+}
+
 // ColoredPrintf does printf in colored way + newline
 func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 	if RunningOnTerminal() {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks
-		var prev rune
+		var inColorMark, inCurly bool
 		msg = strings.Map(func(r rune) rune {
-			if prev == '@' {
-				prev = 0
-				if r == '@' {
-					return r
+			if inColorMark {
+				if inCurly {
+					if r == '}' {
+						inCurly = false
+						inColorMark = false
+						return -1
+					}
+				} else {
+					if r == '{' {
+						inCurly = true
+					} else if r == '@' {
+						return '@'
+					} else {
+						inColorMark = false
+					}
 				}
 				return -1
 			}
-			prev = r
-			if r == '@' {
-				return -1
 
+			if r == '@' {
+				inColorMark = true
+				return -1
 			}
 
 			return r
@@ -151,17 +174,29 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 }
 
 func (p *Progress) worker() {
+	hasBar := false
+
 	for {
 		task := <-p.queue
 		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
 		case codePrint:
 			if p.barShown {
 				fmt.Print("\r\033[2K")
 				p.barShown = false
 			}
 			fmt.Print(task.message)
+		case codePrintStdErr:
+			if p.barShown {
+				fmt.Print("\r\033[2K")
+				p.barShown = false
+			}
+			fmt.Fprint(os.Stderr, task.message)
 		case codeProgress:
-			if p.bar != nil {
+			if hasBar {
 				fmt.Print("\r" + task.message)
 				p.barShown = true
 			}

console/terminal.go

@@ -1,10 +1,9 @@
-// +build !freebsd
-
 package console
 
 import (
-	"code.google.com/p/go.crypto/ssh/terminal"
 	"syscall"
+
+	"golang.org/x/crypto/ssh/terminal"
 )
 
 // RunningOnTerminal checks whether stdout is terminal

console/terminal_bsd.go

@@ -1,10 +0,0 @@
-// +build freebsd
-
-package console
-
-// RunningOnTerminal checks whether stdout is terminal
-//
-// Stub for FreeBSD, until in go1.3 terminal.IsTerminal would start working for FreeBSD
-func RunningOnTerminal() bool {
-	return false
-}

context/context.go

@@ -0,0 +1,595 @@
+// Package context provides single entry to all resources
+package context
+
+import (
+	gocontext "context"
+	"fmt"
+	"math/rand"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/http"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/s3"
+	"github.com/aptly-dev/aptly/swift"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+// AptlyContext is a common context shared by all commands
+type AptlyContext struct {
+	sync.Mutex
+
+	gocontext.Context
+
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
+
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorages map[string]aptly.PublishedStorage
+	collectionFactory *deb.CollectionFactory
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// Check interface
+var _ aptly.PublishedStorageProvider = &AptlyContext{}
+
+// FatalError is type for panicking to abort execution with non-zero
+// exit code and print meaningful explanation
+type FatalError struct {
+	ReturnCode int
+	Message    string
+}
+
+// Fatal panics and aborts execution with exit code 1
+func Fatal(err error) {
+	returnCode := 1
+	if err == commander.ErrFlagError || err == commander.ErrCommandError {
+		returnCode = 2
+	}
+	panic(&FatalError{ReturnCode: returnCode, Message: err.Error()})
+}
+
+// Config loads and returns current configuration
+func (context *AptlyContext) Config() *utils.ConfigStructure {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.config()
+}
+
+func (context *AptlyContext) config() *utils.ConfigStructure {
+	if !context.configLoaded {
+		var err error
+
+		configLocation := context.globalFlags.Lookup("config").Value.String()
+		if configLocation != "" {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			configLocations := []string{
+				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+				"/etc/aptly.conf",
+			}
+
+			for _, configLocation := range configLocations {
+				err = utils.LoadConfig(configLocation, &utils.Config)
+				if err == nil {
+					break
+				}
+				if !os.IsNotExist(err) {
+					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				}
+			}
+
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+
+				// as this is fresh aptly installation, we don't need to support legacy pool locations
+				utils.Config.SkipLegacyPool = true
+				utils.SaveConfig(configLocations[0], &utils.Config)
+			}
+		}
+
+		context.configLoaded = true
+
+	}
+	return &utils.Config
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func (context *AptlyContext) LookupOption(defaultValue bool, name string) (result bool) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.lookupOption(defaultValue, name)
+}
+
+func (context *AptlyContext) lookupOption(defaultValue bool, name string) (result bool) {
+	result = defaultValue
+
+	if context.globalFlags.IsSet(name) {
+		result = context.globalFlags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// DependencyOptions calculates options related to dependecy handling
+func (context *AptlyContext) DependencyOptions() int {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.dependencyOptions == -1 {
+		context.dependencyOptions = 0
+		if context.lookupOption(context.config().DepFollowSuggests, "dep-follow-suggests") {
+			context.dependencyOptions |= deb.DepFollowSuggests
+		}
+		if context.lookupOption(context.config().DepFollowRecommends, "dep-follow-recommends") {
+			context.dependencyOptions |= deb.DepFollowRecommends
+		}
+		if context.lookupOption(context.config().DepFollowAllVariants, "dep-follow-all-variants") {
+			context.dependencyOptions |= deb.DepFollowAllVariants
+		}
+		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
+			context.dependencyOptions |= deb.DepFollowSource
+		}
+		if context.lookupOption(context.config().DepVerboseResolve, "dep-verbose-resolve") {
+			context.dependencyOptions |= deb.DepVerboseResolve
+		}
+	}
+
+	return context.dependencyOptions
+}
+
+// ArchitecturesList returns list of architectures fixed via command line or config
+func (context *AptlyContext) ArchitecturesList() []string {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.architecturesList == nil {
+		context.architecturesList = context.config().Architectures
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
+		if optionArchitectures != "" {
+			context.architecturesList = strings.Split(optionArchitectures, ",")
+		}
+	}
+
+	return context.architecturesList
+}
+
+// Progress creates or returns Progress object
+func (context *AptlyContext) Progress() aptly.Progress {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._progress()
+}
+
+func (context *AptlyContext) _progress() aptly.Progress {
+	if context.progress == nil {
+		context.progress = console.NewProgress()
+		context.progress.Start()
+	}
+
+	return context.progress
+}
+
+// Downloader returns instance of current downloader
+func (context *AptlyContext) Downloader() aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader == nil {
+		var downloadLimit int64
+		limitFlag := context.flags.Lookup("download-limit")
+		if limitFlag != nil {
+			downloadLimit = limitFlag.Value.Get().(int64)
+		}
+		if downloadLimit == 0 {
+			downloadLimit = context.config().DownloadLimit
+		}
+		context.downloader = http.NewDownloader(downloadLimit*1024, context._progress())
+	}
+
+	return context.downloader
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) DBPath() string {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.dbPath()
+}
+
+// DBPath builds path to database
+func (context *AptlyContext) dbPath() string {
+	return filepath.Join(context.config().RootDir, "db")
+}
+
+// Database opens and returns current instance of database
+func (context *AptlyContext) Database() (database.Storage, error) {
+	context.Lock()
+	defer context.Unlock()
+
+	return context._database()
+}
+
+func (context *AptlyContext) _database() (database.Storage, error) {
+	if context.database == nil {
+		var err error
+
+		context.database, err = database.NewDB(context.dbPath())
+		if err != nil {
+			return nil, fmt.Errorf("can't instantiate database: %s", err)
+		}
+	}
+
+	tries := context.flags.Lookup("db-open-attempts").Value.Get().(int)
+	const BaseDelay = 10 * time.Second
+	const Jitter = 1 * time.Second
+
+	for ; tries >= 0; tries-- {
+		err := context.database.Open()
+		if err == nil || !strings.Contains(err.Error(), "resource temporarily unavailable") {
+			return context.database, err
+		}
+
+		if tries > 0 {
+			delay := time.Duration(rand.NormFloat64()*float64(Jitter) + float64(BaseDelay))
+			if delay < 0 {
+				delay = time.Second
+			}
+
+			context._progress().PrintfStdErr("Unable to open database, sleeping %s, attempts left %d...\n", delay, tries)
+			time.Sleep(delay)
+		}
+	}
+
+	return nil, fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	_, err := context.Database()
+
+	return err
+}
+
+// CollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.collectionFactory == nil {
+		db, err := context._database()
+		if err != nil {
+			Fatal(err)
+		}
+		context.collectionFactory = deb.NewCollectionFactory(db)
+	}
+
+	return context.collectionFactory
+}
+
+// PackagePool returns instance of PackagePool
+func (context *AptlyContext) PackagePool() aptly.PackagePool {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.packagePool == nil {
+		context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
+	}
+
+	return context.packagePool
+}
+
+// GetPublishedStorage returns instance of PublishedStorage
+func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+	context.Lock()
+	defer context.Unlock()
+
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
+		if name == "" {
+			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
+		} else if strings.HasPrefix(name, "filesystem:") {
+			params, ok := context.config().FileSystemPublishRoots[name[11:]]
+			if !ok {
+				Fatal(fmt.Errorf("published local storage %v not configured", name[11:]))
+			}
+
+			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
+		} else if strings.HasPrefix(name, "s3:") {
+			params, ok := context.config().S3PublishRoots[name[3:]]
+			if !ok {
+				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+			}
+
+			var err error
+			publishedStorage, err = s3.NewPublishedStorage(
+				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
+				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
+				params.ForceSigV2, params.Debug)
+			if err != nil {
+				Fatal(err)
+			}
+		} else if strings.HasPrefix(name, "swift:") {
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
+				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+		}
+		context.publishedStorages[name] = publishedStorage
+	}
+
+	return publishedStorage
+}
+
+// UploadPath builds path to upload storage
+func (context *AptlyContext) UploadPath() string {
+	return filepath.Join(context.Config().RootDir, "upload")
+}
+
+func (context *AptlyContext) pgpProvider() string {
+	var provider string
+
+	if context.globalFlags.IsSet("gpg-provider") {
+		provider = context.globalFlags.Lookup("gpg-provider").Value.String()
+	} else {
+		provider = context.config().GpgProvider
+	}
+
+	switch provider {
+	case "gpg": // nolint: goconst
+	case "gpg1": // nolint: goconst
+	case "gpg2": // nolint: goconst
+	case "internal": // nolint: goconst
+	default:
+		Fatal(fmt.Errorf("unknown gpg provider: %v", provider))
+	}
+
+	return provider
+}
+
+func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
+	switch context.pgpProvider() {
+	case "gpg1":
+		return pgp.GPG1Finder()
+	case "gpg2":
+		return pgp.GPG2Finder()
+	case "gpg":
+		return pgp.GPGDefaultFinder()
+	}
+
+	panic("uknown GPG provider type")
+}
+
+// GetSigner returns Signer with respect to provider
+func (context *AptlyContext) GetSigner() pgp.Signer {
+	context.Lock()
+	defer context.Unlock()
+
+	provider := context.pgpProvider()
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoSigner{}
+	}
+
+	return pgp.NewGpgSigner(context.getGPGFinder(provider))
+}
+
+// GetVerifier returns Verifier with respect to provider
+func (context *AptlyContext) GetVerifier() pgp.Verifier {
+	context.Lock()
+	defer context.Unlock()
+
+	provider := context.pgpProvider()
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoVerifier{}
+	}
+
+	return pgp.NewGpgVerifier(context.getGPGFinder(provider))
+}
+
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.Lock()
+	defer context.Unlock()
+
+	context.flags = flags
+}
+
+// Flags returns current command flags
+func (context *AptlyContext) Flags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.flags
+}
+
+// GlobalFlags returns flags passed to all commands
+func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.globalFlags
+}
+
+// GoContextHandleSignals upgrades context to handle ^C by aborting context
+func (context *AptlyContext) GoContextHandleSignals() {
+	context.Lock()
+	defer context.Unlock()
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	var cancel gocontext.CancelFunc
+
+	context.Context, cancel = gocontext.WithCancel(context.Context)
+
+	go func() {
+		<-sigch
+		signal.Stop(sigch)
+		context.Progress().PrintfStdErr("Aborting... press ^C once again to abort immediately\n")
+		cancel()
+	}()
+}
+
+// Shutdown shuts context down
+func (context *AptlyContext) Shutdown() {
+	context.Lock()
+	defer context.Unlock()
+
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+		context.database = nil
+	}
+	if context.downloader != nil {
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// Cleanup does partial shutdown of context
+func (context *AptlyContext) Cleanup() {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.downloader != nil {
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// NewContext initializes context with default settings
+func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
+	var err error
+
+	context := &AptlyContext{
+		flags:             flags,
+		globalFlags:       flags,
+		dependencyOptions: -1,
+		Context:           gocontext.TODO(),
+		publishedStorages: map[string]aptly.PublishedStorage{},
+	}
+
+	if aptly.EnableDebug {
+		cpuprofile := flags.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return nil, err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := flags.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		memstats := flags.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return nil, err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return context, nil
+}

context/context_test.go

@@ -0,0 +1,86 @@
+package context
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type fatalErrorPanicChecker struct {
+	*CheckerInfo
+}
+
+var FatalErrorPanicMatches Checker = &fatalErrorPanicChecker{
+	&CheckerInfo{Name: "FatalErrorPanics", Params: []string{"function", "expected"}},
+}
+
+func (checker *fatalErrorPanicChecker) Check(params []interface{}, names []string) (result bool, errmsg string) {
+	f := reflect.ValueOf(params[0])
+	if f.Kind() != reflect.Func || f.Type().NumIn() != 0 {
+		return false, "Function must take zero arguments"
+	}
+	defer func() {
+		if errmsg != "" {
+			return
+		}
+
+		obtained := recover()
+
+		names[0] = "panic"
+
+		var ok bool
+
+		var e1 *FatalError
+		if e1, ok = obtained.(*FatalError); ok {
+			params[0] = e1
+		} else {
+			errmsg = "Panic value is not FatalError"
+			return
+		}
+
+		var e2 *FatalError
+		if e2, ok = params[1].(*FatalError); ok {
+			params[1] = e2
+		} else {
+			errmsg = "Expected value is not FatalError"
+			return
+		}
+
+		if *e1 == *e2 {
+			result = true
+		} else {
+			result = false
+			errmsg = "Not equal"
+		}
+	}()
+	f.Call(nil)
+	return false, "Function has not panicked"
+}
+
+type AptlyContextSuite struct {
+	context *AptlyContext
+}
+
+var _ = Suite(&AptlyContextSuite{})
+
+func (s *AptlyContextSuite) SetUpTest(c *C) {
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.String("config", "", "")
+	context, err := NewContext(flags)
+	c.Assert(err, IsNil)
+	s.context = context
+}
+
+func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
+	// https://github.com/aptly-dev/aptly/issues/711
+	// This will fail on account of us not having a config, so the
+	// storage never exists.
+	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
+		FatalErrorPanicMatches,
+		&FatalError{ReturnCode: 1, Message: "published local storage fuji not configured"})
+}

database/leveldb.go

@@ -4,6 +4,9 @@ package database
 import (
 	"bytes"
 	"errors"
+	"io/ioutil"
+	"os"
+
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -16,20 +19,29 @@ var (
 	ErrNotFound = errors.New("key not found")
 )
 
+// StorageProcessor is a function to process one single storage entry
+type StorageProcessor func(key []byte, value []byte) error
+
 // Storage is an interface to KV storage
 type Storage interface {
+	CreateTemporary() (Storage, error)
 	Get(key []byte) ([]byte, error)
 	Put(key []byte, value []byte) error
 	Delete(key []byte) error
+	HasPrefix(prefix []byte) bool
+	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
+	Open() error
 	Close() error
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
+	Drop() error
 }
 
 type levelDB struct {
+	path  string
 	db    *leveldb.DB
 	batch *leveldb.Batch
 }
@@ -39,22 +51,39 @@ var (
 	_ Storage = &levelDB{}
 )
 
-// OpenDB opens (creates) LevelDB database
-func OpenDB(path string) (Storage, error) {
+func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	o := &opt.Options{
-		Filter: filter.NewBloomFilter(10),
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
 	}
 
-	db, err := leveldb.OpenFile(path, o)
+	if throttleCompaction {
+		o.CompactionL0Trigger = 32
+		o.WriteL0PauseTrigger = 96
+		o.WriteL0SlowdownTrigger = 64
+	}
+
+	return leveldb.OpenFile(path, o)
+}
+
+// NewDB creates new instance of DB, but doesn't open it (yet)
+func NewDB(path string) (Storage, error) {
+	return &levelDB{path: path}, nil
+}
+
+// NewOpenDB creates new instance of DB and opens it
+func NewOpenDB(path string) (Storage, error) {
+	db, err := NewDB(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db}, nil
+
+	return db, db.Open()
 }
 
 // RecoverDB recovers LevelDB database from corruption
 func RecoverDB(path string) error {
-	stor, err := storage.OpenFile(path)
+	stor, err := storage.OpenFile(path, false)
 	if err != nil {
 		return err
 	}
@@ -70,6 +99,20 @@ func RecoverDB(path string) error {
 	return nil
 }
 
+// CreateTemporary creates new DB of the same type in temp dir
+func (l *levelDB) CreateTemporary() (Storage, error) {
+	tempdir, err := ioutil.TempDir("", "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := internalOpen(tempdir, true)
+	if err != nil {
+		return nil, err
+	}
+	return &levelDB{db: db, path: tempdir}, nil
+}
+
 // Get key value from database
 func (l *levelDB) Get(key []byte) ([]byte, error) {
 	value, err := l.db.Get(key, nil)
@@ -95,7 +138,7 @@ func (l *levelDB) Put(key []byte, value []byte) error {
 			return err
 		}
 	} else {
-		if bytes.Compare(old, value) == 0 {
+		if bytes.Equal(old, value) {
 			return nil
 		}
 	}
@@ -145,9 +188,48 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	return result
 }
 
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (l *levelDB) HasPrefix(prefix []byte) bool {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (l *levelDB) ProcessByPrefix(prefix []byte, proc StorageProcessor) error {
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		err := proc(iterator.Key(), iterator.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Close finishes DB work
 func (l *levelDB) Close() error {
-	return l.db.Close()
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
+}
+
+// Reopen tries to open (re-open) the database
+func (l *levelDB) Open() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path, false)
+	return err
 }
 
 // StartBatch starts batch processing of keys
@@ -174,3 +256,12 @@ func (l *levelDB) FinishBatch() error {
 func (l *levelDB) CompactDB() error {
 	return l.db.CompactRange(util.Range{})
 }
+
+// Drop removes all the DB files (DANGEROUS!)
+func (l *levelDB) Drop() error {
+	if l.db != nil {
+		return errors.New("DB is still open")
+	}
+
+	return os.RemoveAll(l.path)
+}

database/leveldb_test.go

@@ -1,8 +1,9 @@
 package database
 
 import (
-	. "launchpad.net/gocheck"
 	"testing"
+
+	. "gopkg.in/check.v1"
 )
 
 // Launch gocheck tests
@@ -21,7 +22,7 @@ func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
 	s.path = c.MkDir()
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -45,7 +46,7 @@ func (s *LevelDBSuite) TestRecoverDB(c *C) {
 	err = RecoverDB(s.path)
 	c.Check(err, IsNil)
 
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Check(err, IsNil)
 
 	result, err := s.db.Get(key)
@@ -70,6 +71,29 @@ func (s *LevelDBSuite) TestGetPut(c *C) {
 	c.Assert(result, DeepEquals, value)
 }
 
+func (s *LevelDBSuite) TestTemporaryDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	temp, err := s.db.CreateTemporary()
+	c.Assert(err, IsNil)
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, false)
+
+	err = temp.Put(key, value)
+	c.Assert(err, IsNil)
+	c.Check(temp.HasPrefix([]byte(nil)), Equals, true)
+
+	c.Assert(temp.Close(), IsNil)
+	c.Assert(temp.Drop(), IsNil)
+}
+
 func (s *LevelDBSuite) TestDelete(c *C) {
 	var (
 		key   = []byte("key")
@@ -106,10 +130,41 @@ func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return ErrNotFound
+	}), Equals, ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return ErrNotFound
+	}), IsNil)
+
 	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
 	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
 }
 
+func (s *LevelDBSuite) TestHasPrefix(c *C) {
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
 func (s *LevelDBSuite) TestBatch(c *C) {
 	var (
 		key    = []byte("key")
@@ -155,3 +210,23 @@ func (s *LevelDBSuite) TestCompactDB(c *C) {
 
 	c.Check(s.db.CompactDB(), IsNil)
 }
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.Open()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/changes.go

@@ -0,0 +1,432 @@
+package deb
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
+)
+
+// Changes is a result of .changes file parsing
+type Changes struct {
+	Changes               string
+	Distribution          string
+	Files                 PackageFiles
+	BasePath, ChangesName string
+	TempDir               string
+	Source                string
+	Binary                []string
+	Architectures         []string
+	Stanza                Stanza
+	SignatureKeys         []pgp.Key
+}
+
+// NewChanges moves .changes file into temporary directory and creates Changes structure
+func NewChanges(path string) (*Changes, error) {
+	var err error
+
+	c := &Changes{
+		BasePath:    filepath.Dir(path),
+		ChangesName: filepath.Base(path),
+	}
+
+	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	// copy .changes file into temporary directory
+	err = utils.CopyFile(filepath.Join(c.BasePath, c.ChangesName), filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// VerifyAndParse does optional signature verification and parses changes files
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier pgp.Verifier) error {
+	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
+	if err != nil {
+		return err
+	}
+	defer input.Close()
+
+	isClearSigned, err := verifier.IsClearSigned(input)
+	if err != nil {
+		return err
+	}
+
+	input.Seek(0, 0)
+
+	if !isClearSigned && !acceptUnsigned {
+		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
+	}
+
+	if isClearSigned && !ignoreSignature {
+		var keyInfo *pgp.KeyInfo
+		keyInfo, err = verifier.VerifyClearsigned(input, false)
+		if err != nil {
+			return err
+		}
+		input.Seek(0, 0)
+
+		c.SignatureKeys = keyInfo.GoodKeys
+	}
+
+	var text io.ReadCloser
+
+	if isClearSigned {
+		text, err = verifier.ExtractClearsigned(input)
+		if err != nil {
+			return err
+		}
+		defer text.Close()
+	} else {
+		text = input
+	}
+
+	reader := NewControlFileReader(text, false, false)
+	c.Stanza, err = reader.ReadStanza()
+	if err != nil {
+		return err
+	}
+
+	c.Distribution = c.Stanza["Distribution"]
+	c.Changes = c.Stanza["Changes"]
+	c.Source = c.Stanza["Source"]
+	c.Binary = strings.Fields(c.Stanza["Binary"])
+	c.Architectures = strings.Fields(c.Stanza["Architecture"])
+
+	c.Files, err = c.Files.ParseSumFields(c.Stanza)
+	return err
+}
+
+// Prepare creates temporary directory, copies file there and verifies checksums
+func (c *Changes) Prepare() error {
+	var err error
+
+	for _, file := range c.Files {
+		if filepath.Dir(file.Filename) != "." {
+			return fmt.Errorf("file is not in the same folder as .changes file: %s", file.Filename)
+		}
+
+		file.Filename = filepath.Base(file.Filename)
+
+		err = utils.CopyFile(filepath.Join(c.BasePath, file.Filename), filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, file := range c.Files {
+		var info utils.ChecksumInfo
+
+		info, err = utils.ChecksumsForFile(filepath.Join(c.TempDir, file.Filename))
+		if err != nil {
+			return err
+		}
+
+		if info.Size != file.Checksums.Size {
+			return fmt.Errorf("size mismatch: expected %v != obtained %v", file.Checksums.Size, info.Size)
+		}
+
+		if info.MD5 != file.Checksums.MD5 {
+			return fmt.Errorf("checksum mismatch MD5: expected %v != obtained %v", file.Checksums.MD5, info.MD5)
+		}
+
+		if info.SHA1 != file.Checksums.SHA1 {
+			return fmt.Errorf("checksum mismatch SHA1: expected %v != obtained %v", file.Checksums.SHA1, info.SHA1)
+		}
+
+		if info.SHA256 != file.Checksums.SHA256 {
+			return fmt.Errorf("checksum mismatch SHA256 expected %v != obtained %v", file.Checksums.SHA256, info.SHA256)
+		}
+	}
+
+	return nil
+}
+
+// Cleanup removes all temporary files
+func (c *Changes) Cleanup() error {
+	if c.TempDir == "" {
+		return nil
+	}
+
+	return os.RemoveAll(c.TempDir)
+}
+
+// PackageQuery returns query that every package should match to be included
+func (c *Changes) PackageQuery() PackageQuery {
+	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
+	for _, arch := range c.Architectures {
+		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// if c.Source is empty, this would never match
+	sourceQuery := &AndQuery{
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource},
+		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
+	}
+
+	var binaryQuery PackageQuery
+	if len(c.Binary) > 0 {
+		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
+		// matching debug ddeb packages, they're not present in the Binary field
+		var ddebQuery PackageQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
+
+		for _, binary := range c.Binary[1:] {
+			binaryQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: binary},
+				R: binaryQuery,
+			}
+			ddebQuery = &OrQuery{
+				L: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", binary)},
+				R: ddebQuery,
+			}
+		}
+
+		ddebQuery = &AndQuery{
+			L: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: c.Source},
+			R: ddebQuery,
+		}
+
+		binaryQuery = &OrQuery{
+			L: binaryQuery,
+			R: ddebQuery,
+		}
+
+		binaryQuery = &AndQuery{
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource}},
+			R: binaryQuery}
+	}
+
+	var nameQuery PackageQuery
+	if binaryQuery == nil {
+		nameQuery = sourceQuery
+	} else {
+		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
+	}
+
+	return &AndQuery{L: archQuery, R: nameQuery}
+}
+
+// GetField implements PackageLike interface
+func (c *Changes) GetField(field string) string {
+	return c.Stanza[field]
+}
+
+// MatchesDependency implements PackageLike interface
+func (c *Changes) MatchesDependency(d Dependency) bool {
+	return false
+}
+
+// MatchesArchitecture implements PackageLike interface
+func (c *Changes) MatchesArchitecture(arch string) bool {
+	return false
+}
+
+// GetName implements PackageLike interface
+func (c *Changes) GetName() string {
+	return ""
+}
+
+// GetVersion implements PackageLike interface
+func (c *Changes) GetVersion() string {
+	return ""
+
+}
+
+// GetArchitecture implements PackageLike interface
+func (c *Changes) GetArchitecture() string {
+	return ""
+}
+
+// CollectChangesFiles walks filesystem collecting all .changes files
+func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	for _, location := range locations {
+		info, err2 := os.Stat(location)
+		if err2 != nil {
+			reporter.Warning("Unable to process %s: %s", location, err2)
+			failedFiles = append(failedFiles, location)
+			continue
+		}
+		if info.IsDir() {
+			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+				if err3 != nil {
+					return err3
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFiles = append(changesFiles, path)
+				}
+
+				return nil
+			})
+
+			if err2 != nil {
+				reporter.Warning("Unable to process %s: %s", location, err2)
+				failedFiles = append(failedFiles, location)
+				continue
+			}
+		} else if strings.HasSuffix(info.Name(), ".changes") {
+			changesFiles = append(changesFiles, location)
+		}
+	}
+
+	sort.Strings(changesFiles)
+
+	return
+}
+
+// ImportChangesFiles imports referenced files in changes files into local repository
+func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
+	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
+	}
+	for _, path := range changesFiles {
+		var changes *Changes
+
+		changes, err = NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		if progress != nil {
+			progress.Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+		}
+
+		var repo *LocalRepo
+		repo, err = localRepoCollection.ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = parseQuery(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return nil, nil, fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		var list *PackageList
+		list, err = NewPackageListFromRefList(repo.RefList(), packageCollection, progress)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, otherFiles, _ := CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		restriction := changes.PackageQuery()
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
+			packageCollection, reporter, restriction, checksumStorage)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
+
+		err = localRepoCollection.Update(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err = os.Remove(file)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	return processedFiles, failedFiles, nil
+}

deb/changes_test.go

@@ -0,0 +1,152 @@
+package deb
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type ChangesSuite struct {
+	Dir, Path           string
+	Reporter            aptly.ResultReporter
+	db                  database.Storage
+	localRepoCollection *LocalRepoCollection
+	packageCollection   *PackageCollection
+	packagePool         aptly.PackagePool
+	checksumStorage     aptly.ChecksumStorage
+	progress            aptly.Progress
+}
+
+var _ = Suite(&ChangesSuite{})
+
+func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Reporter = &aptly.RecordingResultReporter{
+		Warnings:     []string{},
+		AddedLines:   []string{},
+		RemovedLines: []string{},
+	}
+	s.Dir = c.MkDir()
+	s.Path = filepath.Join(s.Dir, "calamares.changes")
+	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
+	c.Assert(err, IsNil)
+
+	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.localRepoCollection = NewLocalRepoCollection(s.db)
+	s.packageCollection = NewPackageCollection(s.db)
+
+	s.checksumStorage = files.NewMockChecksumStorage()
+	s.packagePool = files.NewPackagePool(s.Dir, false)
+	s.progress = console.NewProgress()
+	s.progress.Start()
+}
+
+func (s *ChangesSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
+	s.db.Close()
+}
+
+func (s *ChangesSuite) TestParseAndVerify(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	c.Check(changes.Distribution, Equals, "sid")
+	c.Check(changes.Files, HasLen, 4)
+	c.Check(changes.Files[0].Filename, Equals, "calamares_0+git20141127.99.dsc")
+	c.Check(changes.Files[0].Checksums.Size, Equals, int64(1106))
+	c.Check(changes.Files[0].Checksums.MD5, Equals, "05fd8f3ffe8f362c5ef9bad2f936a56e")
+	c.Check(changes.Files[0].Checksums.SHA1, Equals, "79f10e955dab6eb25b7f7bae18213f367a3a0396")
+	c.Check(changes.Files[0].Checksums.SHA256, Equals, "35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc")
+	c.Check(changes.BasePath, Equals, s.Dir)
+	c.Check(changes.Architectures, DeepEquals, []string{"source", "amd64"})
+	c.Check(changes.Source, Equals, "calamares")
+	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
+}
+
+func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
+	changesFiles, failedFiles := CollectChangesFiles([]string{"testdata/changes"}, s.Reporter)
+
+	c.Check(failedFiles, HasLen, 0)
+	c.Check(changesFiles, DeepEquals, []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	})
+}
+
+func (s *ChangesSuite) TestImportChangesFiles(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	origFailedFiles := []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.0_i386.deb",
+	}
+	origProcessedFiles := []string{
+		"testdata/changes/hardlink_0.2.1.dsc",
+		"testdata/changes/hardlink_0.2.1.tar.gz",
+		"testdata/changes/hardlink_0.2.1_amd64.deb",
+		"testdata/changes/hardlink_0.2.1_amd64.buildinfo",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	}
+
+	var expectedProcessedFiles, expectedFailedFiles []string
+
+	for _, path := range origFailedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedFailedFiles = append(expectedFailedFiles, filename)
+	}
+
+	for _, path := range origProcessedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedProcessedFiles = append(expectedProcessedFiles, filename)
+	}
+
+	changesFiles, failedFiles := CollectChangesFiles([]string{s.Dir}, s.Reporter)
+	c.Check(failedFiles, HasLen, 0)
+
+	processedFiles, failedFiles, err := ImportChangesFiles(
+		append(changesFiles, "testdata/changes/notexistent.changes"),
+		s.Reporter, true, true, false, false, &NullVerifier{},
+		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
+	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
+}
+
+func (s *ChangesSuite) TestPrepare(c *C) {
+	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
+	c.Assert(err, IsNil)
+	err = changes.Prepare()
+	c.Assert(err, IsNil)
+
+	_, err = os.Stat(filepath.Join(changes.TempDir, "hardlink_0.2.1_amd64.changes"))
+	c.Check(err, IsNil)
+}
+
+func (s *ChangesSuite) TestPackageQuery(c *C) {
+	changes, err := NewChanges(s.Path)
+	c.Assert(err, IsNil)
+
+	err = changes.VerifyAndParse(true, true, &NullVerifier{})
+	c.Check(err, IsNil)
+
+	q := changes.PackageQuery()
+	c.Check(q.String(), Equals,
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
+}

deb/checksum_collection.go

@@ -0,0 +1,67 @@
+package deb
+
+import (
+	"bytes"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/ugorji/go/codec"
+)
+
+// ChecksumCollection does management of ChecksumInfo in DB
+type ChecksumCollection struct {
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
+}
+
+// NewChecksumCollection creates new ChecksumCollection and binds it to database
+func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+	return &ChecksumCollection{
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
+	}
+}
+
+func (collection *ChecksumCollection) dbKey(path string) []byte {
+	return []byte("C" + path)
+}
+
+// Get finds checksums in DB by path
+func (collection *ChecksumCollection) Get(path string) (*utils.ChecksumInfo, error) {
+	encoded, err := collection.db.Get(collection.dbKey(path))
+	if err != nil {
+		if err == database.ErrNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	c := &utils.ChecksumInfo{}
+
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+	err = decoder.Decode(c)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Update adds or updates information about checksum in DB
+func (collection *ChecksumCollection) Update(path string, c *utils.ChecksumInfo) error {
+	var encodeBuffer bytes.Buffer
+
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+	err := encoder.Encode(c)
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Put(collection.dbKey(path), encodeBuffer.Bytes())
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &ChecksumCollection{}
+)