S3: support disabling ACL with none value

This change lets you disable ACL when using S3 by using a configuration
value of `none`. This way we maintain backward compatibility with the
default setting being `private`.

Fixes: #1067

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 200
+ignore = E126,E241,E741,W504
+include =
+    system
+exclude =
+    system/env

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
+
+name: CI
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  build:
+    name: test
+    runs-on: ubuntu-20.04
+    continue-on-error: ${{ matrix.allow_failure }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+      matrix:
+        go: [1.16, 1.17, 1.18]
+        run_long_tests: [no]
+        allow_failure: [false]
+        include:
+          # Disable this for now as it's not clear how to select the latest master branch
+          # version of go using actions/setup-go@v2.
+          # - go: master
+          #   run_long_tests: no
+          #   allow_failure: true
+          - go: 1.17
+            run_long_tests: yes
+            allow_failure: false
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.45.0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install O/S packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+
+      - name: Install Python packages
+        run: |
+          pip install six packaging appdirs virtualenv
+          pip install -U pip setuptools
+          pip install -r system/requirements.txt
+
+      - name: Install Azurite
+        id: azuright
+        uses: potatoqualitee/azuright@v1.1
+        with:
+          directory: ${{ runner.temp }}
+
+      - name: Get aptly version
+        run: |
+           make version
+
+      - name: Make
+        env:
+          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
+          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        run: |
+          make
+
+      - name: Upload code coverage
+        if: matrix.run_long_tests
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.txt
+
+  release:
+    name: release
+    needs: build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # fetch the whole repot for `git describe` to
+          # work and get the nightly verion
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+
+      - name: Make Release
+        env:
+          GOBIN: /usr/local/bin
+        run: |
+          make release
+
+      - name: Publish nightly release to aptly
+        if: github.ref == 'refs/heads/master'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh nightly
+
+      - name: Publish release to aptly
+        if: startsWith(github.event.ref, 'refs/tags')
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh release
+
+      - name: Upload artifacts to GitHub Release
+        if: startsWith(github.event.ref, 'refs/tags')
+        uses: softprops/action-gh-release@v1
+        with:
+          body: Release ${{ github.ref }} generated by the CI.
+          files: build/*

.gitignore

@@ -22,13 +22,23 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
-coverage*.out
+coverage.txt
 
 *.pyc
 
-_vendor/
+xc-out/
+root/
 
-gen
 man/aptly.1.html
 man/aptly.1.ronn
+
+.goxc.local.json
+
+system/env/
+
+# created by make build for release artifacts
+build/
+
+pgp/keyrings/aptly2*.gpg
+pgp/keyrings/aptly2*.gpg~
+pgp/keyrings/.#*

.golangci.yml

@@ -0,0 +1,19 @@
+run:
+  tests: false
+
+
+linters:
+  disable-all: true
+  enable:
+   - deadcode
+   - goconst
+   - gofmt
+   - goimports
+   - govet
+   - ineffassign
+   - misspell
+   - revive
+   - staticcheck
+   - structcheck
+   - varcheck
+   - vetshadow

.goxc.json

@@ -0,0 +1,45 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin",
+		"go-test",
+		"go-vet"
+	],
+    "TasksAppend": [
+		"bintray"
+    ],
+    "TaskSettings": {
+        "debs": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainer-email": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Depends": "bzip2, xz-utils, gnupg, gpgv",
+				"Suggests": "graphviz"
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
+	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
+	"MainDirsExclude": "_man,vendor",
+	"BuildSettings": {
+		"LdFlagsXVars": {
+			"Version": "main.Version"
+		}
+	},
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,20 +1,106 @@
+dist: xenial
+sudo: required
+
 language: go
 
-go:
-  - 1.1
-  - 1.2.1
-  - tip
+go_import_path: github.com/aptly-dev/aptly
+
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+    - gnupg2
+    - gpgv2
 
 env:
   global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
+    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+    - NO_FTP_ACCESS: "yes"
+    - BOTO_CONFIG: /dev/null
+    - GO111MODULE: "on"
+    - GOPROXY: https://proxy.golang.org
+
+matrix:
+  allow_failures:
+    - go: master
+      env: RUN_LONG_TESTS=no
+  fast_finish: true
+  include:
+    - go: 1.11.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.12.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.13.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.14.x
+      env: RUN_LONG_TESTS=yes
+    - go: 1.15.x
+      env:
+        - RUN_LONG_TESTS=yes
+        - DEPLOY_BINARIES=yes
+        - APTLY_USER=aptly
+        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
+    - go: master
+      env: RUN_LONG_TESTS=no
+
+before_install:
+  - virtualenv system/env
+  - . system/env/bin/activate
+  - pip install six packaging appdirs
+  - pip install -U pip setuptools
+  - pip install -r system/requirements.txt
+  - make version
 
 install:
   - make prepare
 
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
 
-script: make travis
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false
 
-matrix:
-    allow_failures:
-        - go: tip
+before_deploy:
+  - make release
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
+    file_glob: true
+    file: build/*
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: s3
+    access_key_id:
+      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
+    secret_access_key:
+      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
+    bucket: aptly-nightly
+    skip_cleanup: true
+    acl: public-read
+    local_dir: build
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh nightly
+    skip_cleanup: true
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh release
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -0,0 +1,51 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/queeno)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)
+* Shengjing Zhu (https://github.com/zhsj)
+* Nabil Bendafi (https://github.com/nabilbendafi)
+* Raphael Medaer (https://github.com/rmedaer)
+* Raul Benencia (https://github.com/rul)
+* Don Kuntz (https://github.com/dkuntz2)
+* Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)
+* Markus Muellner (https://github.com/mmianl)
+* Chuan Liu (https://github.com/chuan)
+* Samuel Mutel (https://github.com/smutel)
+* Russell Greene (https://github.com/russelltg)
+* Wade Simmons (https://github.com/wadey)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -0,0 +1,241 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+
+### List of Repositories
+
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly.info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Your First Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+## Development Setup
+
+This section describes local setup to start contributing to aptly source.
+
+### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.
+
+### Forking and Cloning
+
+As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
+
+    git clone git@github.com:aptly-dev/aptly.git
+    cd aptly
+
+For main repo under your GitHub user and add it as another Git remote:
+
+    git remote add <user> git@github.com:<user>/aptly.git
+
+That way you can continue to build project as is (you don't need to adjust import paths), but you would need
+to specify your remote name when pushing branches:
+
+    git push <user> <your-branch>
+
+### Dependencies
+
+You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+
+    make prepare dev system/env
+
+This is usually one-time action.
+
+Aptly is using Go modules to manage dependencies, download modules using:
+
+    make modules
+
+### Building
+
+If you want to build aptly binary from your current source tree, run:
+
+    make install
+
+This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
+
+    aptly
+
+Or, if it's not on your path:
+
+    ~/go/bin/aptly
+
+### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### Style Checks
+
+Style checks could be run with:
+
+    make check
+
+aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
+for the linter could be found in [.golangci.yml](.golangci.yml) file.
+
+Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
+
+### Vendored Code
+
+aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
+repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
+packages over packages in `$GOPATH`.
+
+If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
+Usually all you need is `dep ensure` or `dep ensure -update`.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash and Zsh Completion
+
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.
+
+## Design
+
+This section requires future work.
+
+*TBD*
+
+### Database
+
+### Package Pool
+
+### Package
+
+### PackageList, PackageRefList
+
+### LocalRepo, RemoteRepo, Snapshot
+
+### PublishedRepository
+
+### Context
+
+### Collections, CollectionFactory

Gomfile

@@ -1,22 +0,0 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
-gom 'code.google.com/p/gographviz', :commit => '212766062629'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/smira/commander'
-gom 'github.com/smira/flag'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'ff3719c6816e2cd194f05058452d660608e178ac'
-gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
-gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
-
-group :test do
-    gom 'launchpad.net/gocheck'
-end
-
-group :development do
-   gom 'github.com/golang/lint/golint'
-   gom 'github.com/mattn/goveralls'
-   gom 'github.com/axw/gocov/gocov'
-   gom 'code.google.com/p/go.tools/cmd/cover'
-end

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,75 +1,81 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=database deb files http utils
-ALL_PACKAGES=aptly cmd console database deb files http utils
-BINPATH=$(abspath ./_vendor/bin)
-GOM_ENVIRONMENT=-test
-PYTHON?=python
-
-ifeq ($(GOVERSION), devel)
-TRAVIS_TARGET=coveralls
-GOM_ENVIRONMENT+=-development
+ifdef TRAVIS_TAG
+	TAG=$(TRAVIS_TAG)
 else
-TRAVIS_TARGET=test
+	TAG="$(shell git describe --tags --always)"
 endif
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
+PACKAGES=context database deb files gpg http query swift s3 utils
+PYTHON?=python3
+TESTS?=
+BINPATH?=$(GOPATH)/bin
+RUN_LONG_TESTS?=yes
 
-ifeq ($(TRAVIS), true)
-GOM=$(HOME)/gopath/bin/gom
-else
-GOM=gom
-endif
-
-all: test check system-test
+all: modules test bench check system-test
 
 prepare:
-	go get -u github.com/mattn/gom
-	$(GOM) $(GOM_ENVIRONMENT) install
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
 
-coverage.out:
-	rm -f coverage.*.out
-	for i in $(PACKAGES); do $(GOM) test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-	rm -f coverage.*.out
+modules:
+	go mod download
+	go mod verify
+	go mod tidy -v
 
-coverage: coverage.out
-	$(GOM) exec go tool cover -html=coverage.out
-	rm -f coverage.out
+dev:
+	go get -u github.com/laher/goxc
 
-check:
-	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
-	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
+check: system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	golangci-lint run
+	system/env/bin/flake8
+endif
 
 install:
-	$(GOM) build -o $(BINPATH)/aptly
+	go install -v -ldflags "-X main.Version=$(VERSION)"
 
-system-test: install
-ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
-	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+system/env: system/requirements.txt
+ifeq ($(RUN_LONG_TESTS), yes)
+	rm -rf system/env
+	$(PYTHON) -m venv system/env
+	system/env/bin/pip install -r system/requirements.txt
 endif
-	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
-travis: $(TRAVIS_TARGET) system-test
+system-test: install system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
+endif
 
 test:
-	$(GOM) test -v ./... -gocheck.v=true
+	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
 
-coveralls: coverage.out
-	$(GOM) exec $(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+bench:
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
+goxc: dev
 	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
 	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	cp completion.d/aptly root/etc/bash_completion.d/
+	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
 	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
+	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
 
-.PHONY: coverage.out
+release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
+release: goxc
+	rm -rf build/
+	mkdir -p build/
+	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
+
+man:
+	make -C man
+
+version:
+	@echo $(VERSION)
+
+.PHONY: man modules version release goxc

README.rst

@@ -2,15 +2,24 @@
 aptly
 =====
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
-    :target: https://travis-ci.org/smira/aptly
+.. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/aptly-dev/aptly/actions
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/aptly-dev/aptly
+
+.. image:: https://badges.gitter.im/Join Chat.svg
+    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
+    :target: https://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
 Aptly features: ("+" means planned features)
@@ -20,26 +29,26 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
-* publish self-made packages as Debian repositories (+)
+* filter repository by search query, pulling dependencies when required
+* publish self-made packages as Debian repositories
+* REST API for remote access
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------
 
-To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
     deb http://repo.aptly.info/ squeeze main
 
 And import key that is used to sign the release::
 
-    $ gpg --keyserver keys.gnupg.net --recv-keys 2A194991
-    $ gpg -a --export 2A194991 | sudo apt-key add -
+    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
 
 After that you can install aptly as any other software package::
 
@@ -49,20 +58,60 @@ After that you can install aptly as any other software package::
 Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
 Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
-Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
+If you would like to use nightly builds (unstable), please use following repository::
 
-If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+    deb http://repo.aptly.info/ nightly main
 
-    go get -u github.com/mattn/gom
-    mkdir -p $GOPATH/src/github.com/smira/aptly
-    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
-    cd $GOPATH/src/github.com/smira/aptly
-    gom -production install
-    gom build -o $GOPATH/bin/aptly
+Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
 
-Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
-should work as well, but might fail or produce different result (if external libraries got updated).
+If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
 
-If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+    git clone https://github.com/aptly-dev/aptly
+    cd aptly
+    make modules install
 
+Binary would be installed to ``$GOPATH/bin/aptly``.
 
+Contributing
+------------
+
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+
+Integrations
+------------
+
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
+
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -2,16 +2,17 @@ package main
 
 import (
 	"fmt"
-	"github.com/smira/aptly/cmd"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"text/template"
+
+	"github.com/aptly-dev/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func allFlags(flags *flag.FlagSet) []*flag.Flag {
@@ -43,22 +44,42 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
 	command.Dispatch(nil)
 
-	_, _File, _, _ := runtime.Caller(0)
-	_File, _ = filepath.Abs(_File)
+	_File, _ := filepath.Abs("./man")
 
 	templ := template.New("man").Funcs(template.FuncMap{
 		"allFlags":    allFlags,
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

api/api.go

@@ -0,0 +1,221 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+type dbRequestKind int
+
+const (
+	acquiredb dbRequestKind = iota
+	releasedb
+)
+
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
+var dbRequests chan dbRequest
+
+// Acquire database lock and release it when not needed anymore.
+//
+// Should be run in a goroutine!
+func acquireDatabase() {
+	clients := 0
+	for request := range dbRequests {
+		var err error
+
+		switch request.kind {
+		case acquiredb:
+			if clients == 0 {
+				err = context.ReOpenDatabase()
+			}
+
+			request.err <- err
+
+			if err == nil {
+				clients++
+			}
+		case releasedb:
+			clients--
+			if clients == 0 {
+				err = context.CloseDatabase()
+			} else {
+				err = nil
+			}
+
+			request.err <- err
+		}
+	}
+}
+
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Println("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			c.AbortWithError(409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Println("Executing task synchronously")
+		out := context.Progress()
+		detail := task.Detail{}
+		retValue, err := proc(out, &detail)
+		if err != nil {
+			c.AbortWithError(retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			return
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/api_test.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"encoding/json"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ApiSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+}
+
+var _ = Suite(&ApiSuite{})
+
+func createTestConfig() *os.File {
+	file, err := ioutil.TempFile("", "aptly")
+	if err != nil {
+		return nil
+	}
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+	})
+	if err != nil {
+		return nil
+	}
+	file.Write(jsonString)
+	return file
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	file := createTestConfig()
+	c.Assert(file, NotNil)
+	s.configFile = file
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *ApiSuite) TearDownSuite(c *C) {
+	os.Remove(s.configFile.Name())
+	s.context.Shutdown()
+}
+
+func (s *ApiSuite) SetUpTest(c *C) {
+}
+
+func (s *ApiSuite) TearDownTest(c *C) {
+}
+
+func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w, nil
+}
+
+func (s *ApiSuite) TestGetVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/version", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, ".*Version.*")
+}
+
+func (s *ApiSuite) TestTruthy(c *C) {
+	c.Check(truthy("no"), Equals, false)
+	c.Check(truthy("n"), Equals, false)
+	c.Check(truthy("off"), Equals, false)
+	c.Check(truthy("false"), Equals, false)
+	c.Check(truthy("0"), Equals, false)
+	c.Check(truthy(false), Equals, false)
+	c.Check(truthy(0), Equals, false)
+
+	c.Check(truthy("y"), Equals, true)
+	c.Check(truthy("yes"), Equals, true)
+	c.Check(truthy("t"), Equals, true)
+	c.Check(truthy("true"), Equals, true)
+	c.Check(truthy("1"), Equals, true)
+	c.Check(truthy(true), Equals, true)
+	c.Check(truthy(1), Equals, true)
+
+	c.Check(truthy(nil), Equals, false)
+
+	c.Check(truthy("foobar"), Equals, true)
+	c.Check(truthy(-1), Equals, true)
+	c.Check(truthy(gin.H{}), Equals, true)
+}

api/db.go

@@ -0,0 +1,180 @@
+package api
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/db/cleanup
+func apiDbCleanup(c *gin.Context) {
+
+	resources := []string{string(task.AllResourcesKey)}
+	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var err error
+
+		collectionFactory := context.NewCollectionFactory()
+
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
+
+		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
+		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if e != nil {
+				return e
+			}
+
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+			if published.SourceKind != deb.SourceLocalRepo {
+				return nil
+			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
+
+			for _, component := range published.Components() {
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// ... and compare it to the list of all packages
+		out.Printf("Loading list of all packages...")
+		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+		// delete packages that are no longer referenced
+		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
+
+		// database can't err as collection factory already constructed
+		db, _ := context.Database()
+
+		if toDelete.Len() > 0 {
+			batch := db.CreateBatch()
+			toDelete.ForEach(func(ref []byte) error {
+				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				return nil
+			})
+
+			err = batch.Write()
+			if err != nil {
+				return nil, fmt.Errorf("unable to write to DB: %s", err)
+			}
+		}
+
+		// now, build a list of files that should be present in Repository (package pool)
+		out.Printf("Building list of files referenced by packages...")
+		referencedFiles := make([]string, 0, existingPackageRefs.Len())
+
+		err = existingPackageRefs.ForEach(func(key []byte) error {
+			pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+			if err2 != nil {
+				tail := ""
+				return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+			}
+			paths, err2 := pkg.FilepathList(context.PackagePool())
+			if err2 != nil {
+				return err2
+			}
+			referencedFiles = append(referencedFiles, paths...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sort.Strings(referencedFiles)
+
+		// build a list of files in the package pool
+		out.Printf("Building list of files in package pool...")
+		existingFiles, err := context.PackagePool().FilepathList(out)
+		if err != nil {
+			return nil, fmt.Errorf("unable to collect file paths: %s", err)
+		}
+
+		// find files which are in the pool but not referenced by packages
+		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+		// delete files that are no longer referenced
+		out.Printf("Deleting unreferenced files (%d)...", len(filesToDelete))
+
+		countFilesToDelete := len(filesToDelete)
+		taskDetail := struct {
+			TotalNumberOfPackagesToDelete     int
+			RemainingNumberOfPackagesToDelete int
+		}{
+			countFilesToDelete, countFilesToDelete,
+		}
+		detail.Store(taskDetail)
+
+		if countFilesToDelete > 0 {
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return nil, err
+				}
+
+				taskDetail.RemainingNumberOfPackagesToDelete--
+				detail.Store(taskDetail)
+				totalSize += size
+			}
+
+			out.Printf("Disk space freed: %s...", utils.HumanBytes(totalSize))
+		}
+
+		out.Printf("Compacting database...")
+		return nil, db.CompactDB()
+	})
+}

api/files.go

@@ -0,0 +1,186 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.AbortWithError(404, err)
+		} else {
+			c.AbortWithError(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.AbortWithError(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/gpg.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/gpg
+func apiGPGAddKey(c *gin.Context) {
+	var b struct {
+		Keyserver   string
+		GpgKeyID    string
+		GpgKeyArmor string
+		Keyring     string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	var err error
+	args := []string{"--no-default-keyring"}
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+	args = append(args, "--keyring", keyring)
+	if len(b.Keyserver) > 0 {
+		args = append(args, "--keyserver", b.Keyserver)
+	}
+	if len(b.GpgKeyArmor) > 0 {
+		var tempdir string
+		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+		defer os.RemoveAll(tempdir)
+
+		keypath := filepath.Join(tempdir, "key")
+		keyfile, e := os.Create(keypath)
+		if e != nil {
+			c.AbortWithError(400, e)
+			return
+		}
+		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
+			c.AbortWithError(400, e)
+		}
+		args = append(args, "--import", keypath)
+
+	}
+	if len(b.GpgKeyID) > 0 {
+		keys := strings.Fields(b.GpgKeyID)
+		args = append(args, "--recv-keys")
+		args = append(args, keys...)
+	}
+
+	finder := pgp.GPG1Finder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	// it might happened that we have a situation with an erroneous
+	// gpg command (e.g. when GpgKeyID and GpgKeyArmor is set).
+	// there is no error handling for such as gpg will do this for us
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	cmd.Stdout = os.Stdout
+	if err = cmd.Run(); err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,74 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
+	factory := context.NewCollectionFactory()
+
+	graph, err := deb.BuildGraph(factory, layout)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/middleware.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+)
+
+// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
+// See https://prometheus.io/docs/practices/naming/#labels
+func getBasePath(c *gin.Context) string {
+	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+}
+
+func getURLSegment(url string, idx int) string {
+	var urlSegments = strings.Split(url, "/")
+
+	// Remove segment at index 0 because it's an empty string
+	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
+	return fmt.Sprintf("/%s", segmentAtIndex)
+}
+
+func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		g.WithLabelValues(c.Request.Method, pathFunc(c)).Inc()
+		defer g.WithLabelValues(c.Request.Method, pathFunc(c)).Dec()
+		c.Next()
+	}
+}
+
+func instrumentHandlerCounter(counter *prometheus.CounterVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		counter.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Inc()
+	}
+}
+
+func instrumentHandlerRequestSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(float64(c.Request.ContentLength))
+	}
+}
+
+func instrumentHandlerResponseSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		var responseSize = math.Max(float64(c.Writer.Size()), 0)
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(responseSize)
+	}
+}
+
+func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		now := time.Now()
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
+	}
+}

api/mirror.go

@@ -0,0 +1,556 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
+	if ignoreSignatures {
+		return nil, nil
+	}
+
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring()
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+// GET /api/mirrors
+func apiMirrorsList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	result := []*deb.RemoteRepo{}
+	collection.ForEach(func(repo *deb.RemoteRepo) error {
+		result = append(result, repo)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors
+func apiMirrorsCreate(c *gin.Context) {
+	var err error
+	var b struct {
+		Name               string `binding:"required"`
+		ArchiveURL         string `binding:"required"`
+		Distribution       string
+		Filter             string
+		Components         []string
+		Architectures      []string
+		Keyrings           []string
+		DownloadSources    bool
+		DownloadUdebs      bool
+		DownloadInstaller  bool
+		FilterWithDeps     bool
+		SkipComponentCheck bool
+		IgnoreSignatures   bool
+	}
+
+	b.DownloadSources = context.Config().DownloadSourcePackages
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+	b.Architectures = context.ArchitecturesList()
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
+		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+	}
+
+	if b.Filter != "" {
+		_, err = query.Parse(b.Filter)
+		if err != nil {
+			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			return
+		}
+	}
+
+	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		return
+	}
+
+	repo.Filter = b.Filter
+	repo.FilterWithDeps = b.FilterWithDeps
+	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.DownloadSources = b.DownloadSources
+	repo.DownloadUdebs = b.DownloadUdebs
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	downloader := context.NewDownloader(nil)
+	err = repo.Fetch(downloader, verifier)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		return
+	}
+
+	err = collection.Add(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// DELETE /api/mirrors/:name
+func apiMirrorsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	mirrorCollection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	repo, err := mirrorCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+			}
+		}
+
+		err = mirrorCollection.Drop(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}
+
+// GET /api/mirrors/:name
+func apiMirrorsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/mirrors/:name/packages
+func apiMirrorsPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	if repo.LastDownloadDate.IsZero() {
+		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		return
+	}
+
+	reflist := repo.RefList()
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}
+
+// PUT /api/mirrors/:name
+func apiMirrorsUpdate(c *gin.Context) {
+	var (
+		err    error
+		remote *deb.RemoteRepo
+	)
+
+	var b struct {
+		Name                 string
+		ArchiveURL           string
+		Filter               string
+		Architectures        []string
+		Components           []string
+		Keyrings             []string
+		FilterWithDeps       bool
+		DownloadSources      bool
+		DownloadUdebs        bool
+		SkipComponentCheck   bool
+		IgnoreChecksums      bool
+		IgnoreSignatures     bool
+		ForceUpdate          bool
+		SkipExistingPackages bool
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	remote, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	b.Name = remote.Name
+	b.DownloadUdebs = remote.DownloadUdebs
+	b.DownloadSources = remote.DownloadSources
+	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.FilterWithDeps = remote.FilterWithDeps
+	b.Filter = remote.Filter
+	b.Architectures = remote.Architectures
+	b.Components = remote.Components
+
+	log.Printf("%s: Starting mirror update\n", b.Name)
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Name != remote.Name {
+		_, err = collection.ByName(b.Name)
+		if err == nil {
+			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			return
+		}
+	}
+
+	if b.DownloadUdebs != remote.DownloadUdebs {
+		if remote.IsFlat() && b.DownloadUdebs {
+			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			return
+		}
+	}
+
+	if b.ArchiveURL != "" {
+		remote.SetArchiveRoot(b.ArchiveURL)
+	}
+
+	remote.Name = b.Name
+	remote.DownloadUdebs = b.DownloadUdebs
+	remote.DownloadSources = b.DownloadSources
+	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.FilterWithDeps = b.FilterWithDeps
+	remote.Filter = b.Filter
+	remote.Architectures = b.Architectures
+	remote.Components = b.Components
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	resources := []string{string(remote.Key())}
+	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+		downloader := context.NewDownloader(out)
+		err := remote.Fetch(downloader, verifier)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if !b.ForceUpdate {
+			err = remote.CheckLock()
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if remote.Filter != "" {
+			var filterQuery deb.PackageQuery
+
+			filterQuery, err = query.Parse(remote.Filter)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			_, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		defer func() {
+			// on any interruption, unlock the mirror
+			e := context.ReOpenDatabase()
+			if e == nil {
+				remote.MarkAsIdle()
+				collection.Update(remote)
+			}
+		}()
+
+		remote.MarkAsUpdating()
+		err = collection.Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		context.GoContextHandleSignals()
+
+		count := len(queue)
+		taskDetail := struct {
+			TotalDownloadSize         int64
+			RemainingDownloadSize     int64
+			TotalNumberOfPackages     int
+			RemainingNumberOfPackages int
+		}{
+			downloadSize, downloadSize, count, count,
+		}
+		detail.Store(taskDetail)
+
+		downloadQueue := make(chan int)
+		taskFinished := make(chan *deb.PackageDownloadTask)
+
+		var (
+			errors  []string
+			errLock sync.Mutex
+		)
+
+		pushError := func(err error) {
+			errLock.Lock()
+			errors = append(errors, err.Error())
+			errLock.Unlock()
+		}
+
+		go func() {
+			for idx := range queue {
+				select {
+				case downloadQueue <- idx:
+				case <-context.Done():
+					return
+				}
+			}
+
+			close(downloadQueue)
+		}()
+
+		// update of task details need to be done in order
+		go func() {
+			for {
+				task, ok := <-taskFinished
+				if !ok {
+					return
+				}
+
+				taskDetail.RemainingDownloadSize -= task.File.Checksums.Size
+				taskDetail.RemainingNumberOfPackages--
+				detail.Store(taskDetail)
+			}
+		}()
+
+		log.Printf("%s: Spawning background processes...\n", b.Name)
+		var wg sync.WaitGroup
+		for i := 0; i < context.Config().DownloadConcurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case idx, ok := <-downloadQueue:
+						if !ok {
+							return
+						}
+
+						task := &queue[idx]
+
+						var e error
+
+						// provision download location
+						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// download file...
+						e = context.Downloader().DownloadWithChecksum(
+							context,
+							remote.PackageURL(task.File.DownloadURL()).String(),
+							task.TempDownPath,
+							&task.File.Checksums,
+							b.IgnoreChecksums)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						task.Done = true
+						taskFinished <- task
+					case <-context.Done():
+						return
+					}
+
+				}
+			}()
+		}
+
+		// Wait for all download goroutines to finish
+		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		wg.Wait()
+		log.Printf("%s: Background processes finished\n", b.Name)
+		close(taskFinished)
+
+		for idx := range queue {
+
+			atask := &queue[idx]
+
+			if !atask.Done {
+				// download not finished yet
+				continue
+			}
+
+			// and import it back to the pool
+			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+			}
+
+			// update "attached" files if any
+			for _, additionalAtask := range atask.Additional {
+				additionalAtask.File.PoolPath = atask.File.PoolPath
+				additionalAtask.File.Checksums = atask.File.Checksums
+			}
+		}
+
+		select {
+		case <-context.Done():
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+
+		if len(errors) > 0 {
+			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+		}
+
+		log.Printf("%s: Finalizing download\n", b.Name)
+		remote.FinalizeDownload(collectionFactory, out)
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}

api/mirror_test.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MirrorSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&MirrorSuite{})
+
+func (s *MirrorSuite) TestGetMirrors(c *C) {
+	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}
+
+func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
+	response, _ := s.HTTPRequest("DELETE", "/api/mirrors/does-not-exist", nil)
+	c.Check(response.Code, Equals, 404)
+	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
+}
+
+func (s *MirrorSuite) TestCreateMirror(c *C) {
+	c.ExpectFailure("Need to mock downloads")
+	body, err := json.Marshal(gin.H{
+		"Name":       "dummy",
+		"ArchiveURL": "foobar",
+	})
+	c.Assert(err, IsNil)
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Equals, "")
+}

api/packages.go

@@ -0,0 +1,17 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,395 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (pgp.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := context.GetSigner()
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, collectionFactory)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution         string
+		Label                string
+		Origin               string
+		NotAutomatic         string
+		ButAutomaticUpgrades string
+		ForceOverwrite       bool
+		SkipContents         *bool
+		SkipBz2              *bool
+		Architectures        []string
+		Signing              SigningOptions
+		AcquireByHash        *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var names []string
+	var sources []interface{}
+	var resources []string
+	collectionFactory := context.NewCollectionFactory()
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := collectionFactory.SnapshotCollection()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+			names = append(names, source.Name)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			resources = append(resources, string(snapshot.ResourceKey()))
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == deb.SourceLocalRepo {
+		var localRepo *deb.LocalRepo
+
+		localCollection := collectionFactory.LocalRepoCollection()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+			names = append(names, source.Name)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			resources = append(resources, string(localRepo.Key()))
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	resources = append(resources, string(published.Key()))
+	collection := collectionFactory.PublishedRepoCollection()
+
+	taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskDetail := task.PublishDetail{
+			Detail: detail,
+		}
+		publishOutput := &task.PublishOutput{
+			Progress:      out,
+			PublishDetail: taskDetail,
+		}
+
+		if b.Origin != "" {
+			published.Origin = b.Origin
+		}
+		if b.NotAutomatic != "" {
+			published.NotAutomatic = b.NotAutomatic
+		}
+		if b.ButAutomaticUpgrades != "" {
+			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+		}
+		published.Label = b.Label
+
+		published.SkipContents = context.Config().SkipContentsPublishing
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+
+		published.SkipBz2 = context.Config().SkipBz2Publishing
+		if b.SkipContents != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+
+		duplicate := collection.CheckDuplicate(published)
+		if duplicate != nil {
+			collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
+		}
+
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
+		}
+
+		err = collection.Add(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: published}, nil
+	})
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipContents   *bool
+		SkipBz2        *bool
+		SkipCleanup    *bool
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+		AcquireByHash *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, collectionFactory)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+	var updatedSnapshots []string
+	var resources []string
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		if len(b.Snapshots) > 0 {
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshotCollection := collectionFactory.SnapshotCollection()
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				c.AbortWithError(404, err2)
+				return
+			}
+
+			err2 = snapshotCollection.LoadComplete(snapshot)
+			if err2 != nil {
+				c.AbortWithError(500, err2)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+			updatedSnapshots = append(updatedSnapshots, snapshot.Name)
+		}
+	} else {
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	if b.SkipBz2 != nil {
+		published.SkipBz2 = *b.SkipBz2
+	}
+
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
+	resources = append(resources, string(published.Key()))
+	taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = collection.Update(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		if b.SkipCleanup == nil || !*b.SkipCleanup {
+			err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+				context.GetPublishedStorage(storage), collectionFactory, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
+	})
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(published.Key())}
+
+	taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.Remove(context, storage, prefix, distribution,
+			collectionFactory, out, force, skipCleanup)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}

api/repos.go

@@ -0,0 +1,495 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	collection.ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	err := collection.Add(repo)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Name                *string
+		Comment             *string
+		DefaultDistribution *string
+		DefaultComponent    *string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	if b.Name != nil {
+		_, err := collection.ByName(*b.Name)
+		if err == nil {
+			// already exists
+			c.AbortWithError(404, err)
+			return
+		}
+		repo.Name = *b.Name
+	}
+	if b.Comment != nil {
+		repo.Comment = *b.Comment
+	}
+	if b.DefaultDistribution != nil {
+		repo.DefaultDistribution = *b.DefaultDistribution
+	}
+	if b.DefaultComponent != nil {
+		repo.DefaultComponent = *b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+	name := c.Params.ByName("name")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.ByLocalRepo(repo)
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+	})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList(), collectionFactory)
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Loading packages...\n")
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			var p *deb.Package
+
+			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
+				}
+
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = cb(list, p, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+			}
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Adding package %s\n", p.Name)
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Removing package %s\n", p.Name)
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	dirParam := c.Params.ByName("dir")
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var taskName string
+	var sources []string
+	if fileParam == "" {
+		taskName = fmt.Sprintf("Add packages from dir %s to repo %s", dirParam, name)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+		taskName = fmt.Sprintf("Add package %s from dir %s to repo %s", fileParam, dirParam, name)
+	}
+
+	resources := []string{string(repo.Key())}
+	resources = append(resources, sources...)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		verifier := context.GetVerifier()
+
+		var (
+			packageFiles, failedFiles    []string
+			otherFiles                   []string
+			processedFiles, failedFiles2 []string
+			reporter                     = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+			list *deb.PackageList
+		)
+
+		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+		failedFiles = append(failedFiles, failedFiles2...)
+		processedFiles = append(processedFiles, otherFiles...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+
+		if !noRemove {
+			processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+			for _, file := range processedFiles {
+				err := os.Remove(file)
+				if err != nil {
+					reporter.Warning("unable to remove file %s: %s", file, err)
+				}
+			}
+
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
+	})
+}
+
+// POST /repos/:name/include/:dir/:file
+func apiReposIncludePackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposIncludePackageFromDir(c)
+}
+
+// POST /repos/:name/include/:dir
+func apiReposIncludePackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
+	acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
+	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
+
+	repoTemplateString := c.Params.ByName("name")
+	collectionFactory := context.NewCollectionFactory()
+
+	if !verifyDir(c) {
+		return
+	}
+
+	var sources []string
+	var taskName string
+	dirParam := c.Params.ByName("dir")
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	if fileParam == "" {
+		taskName = fmt.Sprintf("Include packages from changes files in dir %s to repo matching template %s", dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
+	} else {
+		taskName = fmt.Sprintf("Include packages from changes file %s from dir %s to repo matching template %s", fileParam, dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+	}
+
+	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
+		return
+	}
+
+	var resources []string
+	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+		resources = append(resources, task.AllLocalReposResourcesKey)
+	} else {
+		// repo template string is simple text so only use resource key of specific repository
+		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
+
+		resources = append(resources, string(repo.Key()))
+	}
+	resources = append(resources, sources...)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var (
+			err                       error
+			verifier                  = context.GetVerifier()
+			changesFiles              []string
+			failedFiles, failedFiles2 []string
+			reporter                  = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+		)
+
+		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+		_, failedFiles2, err = deb.ImportChangesFiles(
+			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+		failedFiles = append(failedFiles, failedFiles2...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import changes files: %s", err)
+		}
+
+		if !noRemoveFiles {
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
+
+	})
+}

api/router.go

@@ -0,0 +1,163 @@
+package api
+
+import (
+	"net/http"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var context *ctx.AptlyContext
+
+func apiMetricsGet() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
+	}
+}
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	router := gin.Default()
+	router.UseRawPath = true
+	router.Use(gin.ErrorLogger())
+
+	if c.Config().EnableMetricsEndpoint {
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+	}
+
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		dbRequests = make(chan dbRequest)
+
+		go acquireDatabase()
+
+		router.Use(func(c *gin.Context) {
+			var err error
+
+			errCh := make(chan error)
+			dbRequests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			defer func() {
+				dbRequests <- dbRequest{releasedb, errCh}
+				err = <-errCh
+				if err != nil {
+					c.AbortWithError(500, err)
+				}
+			}()
+
+			c.Next()
+		})
+	}
+
+	root := router.Group("/api")
+
+	{
+		if c.Config().EnableMetricsEndpoint {
+			root.GET("/metrics", apiMetricsGet())
+		}
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/mirrors", apiMirrorsList)
+		root.GET("/mirrors/:name", apiMirrorsShow)
+		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		root.POST("/mirrors", apiMirrorsCreate)
+		root.PUT("/mirrors/:name", apiMirrorsUpdate)
+		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+	}
+
+	{
+		root.POST("/gpg/key", apiGPGAddKey)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+	{
+		root.POST("/db/cleanup", apiDbCleanup)
+	}
+	{
+		root.GET("/tasks", apiTasksList)
+		root.POST("/tasks-clear", apiTasksClear)
+		root.GET("/tasks-wait", apiTasksWait)
+		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		root.GET("/tasks/:id/output", apiTasksOutputShow)
+		root.GET("/tasks/:id/detail", apiTasksDetailShow)
+		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		root.GET("/tasks/:id", apiTasksShow)
+		root.DELETE("/tasks/:id", apiTasksDelete)
+		root.POST("/tasks-dummy", apiTasksDummy)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,399 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	repo, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
+		}
+
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	var resources []string
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.AbortWithError(500, err)
+			return
+		}
+
+		resources = append(resources, string(sources[i].ResourceKey()))
+	}
+
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		list := deb.NewPackageList()
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+				}
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = list.Add(p)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+			}
+		}
+
+		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+	})
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	repo, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	snapshot, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		_, err := collection.ByName(b.Name)
+		if err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+		}
+
+		if b.Name != "" {
+			snapshot.Name = b.Name
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: snapshot}, nil
+	})
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(snapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Delete snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.BySnapshot(snapshot)
+
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
+		}
+
+		if !force {
+			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
+			}
+		}
+
+		err = snapshotCollection.Drop(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList(), collectionFactory)
+}

api/task.go

@@ -0,0 +1,155 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /tasks
+func apiTasksList(c *gin.Context) {
+	list := context.TaskList()
+	c.JSON(200, list.GetTasks())
+}
+
+// POST /tasks-clear
+func apiTasksClear(c *gin.Context) {
+	list := context.TaskList()
+	list.Clear()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks-wait
+func apiTasksWait(c *gin.Context) {
+	list := context.TaskList()
+	list.Wait()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks/:id/wait
+func apiTasksWaitForTaskByID(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	task, err := list.WaitForTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id
+func apiTasksShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var task task.Task
+	task, err = list.GetTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id/output
+func apiTasksOutputShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var output string
+	output, err = list.GetTaskOutputByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// GET /tasks/:id/detail
+func apiTasksDetailShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var detail interface{}
+	detail, err = list.GetTaskDetailByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, detail)
+}
+
+// GET /tasks/:id/return_value
+func apiTasksReturnValueShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err := list.GetTaskReturnValueByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// DELETE /tasks/:id
+func apiTasksDelete(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var delTask task.Task
+	delTask, err = list.DeleteTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, delTask)
+}
+
+// POST /tasks-dummy
+func apiTasksDummy(c *gin.Context) {
+	resources := []string{"dummy"}
+	taskName := fmt.Sprintf("Dummy task")
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Dummy task started\n")
+		detail.Store([]int{1, 2, 3})
+		out.Printf("Dummy task finished\n")
+		return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
+	})
+}

api/task_test.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aptly-dev/aptly/task"
+
+	. "gopkg.in/check.v1"
+)
+
+type TaskSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&TaskSuite{})
+
+func (s *TaskSuite) TestTasksDummy(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
+	c.Check(response.Code, Equals, 418)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+}
+
+func (s *TaskSuite) TestTasksDummyAsync(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
+	err = json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
+}
+
+func (s *TaskSuite) TestTaskDelete(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
+	// Give the task time to start
+	time.Sleep(time.Second)
+	response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
+	c.Check(response.Code, Equals, 200)
+}
+
+func (s *TaskSuite) TestTasksClear(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	var ts []task.Task
+	err = json.Unmarshal(response.Body.Bytes(), &ts)
+	c.Assert(err, IsNil)
+	c.Check(len(ts), Equals, 1)
+	c.Check(ts[0].State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "null")
+}

aptly-api.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository API
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/api/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
+
+[Install]
+WantedBy=multi-user.target

aptly.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository server
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/commands/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
+
+[Install]
+WantedBy=multi-user.target

aptly/interfaces.go

@@ -3,49 +3,130 @@
 package aptly
 
 import (
-	"github.com/smira/aptly/utils"
+	"context"
 	"io"
 	"os"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/utils"
 )
 
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
 // PackagePool is asbtraction of package pool storage.
 //
 // PackagePool stores all the package files, deduplicating them.
 type PackagePool interface {
-	// Path returns full path to package file in pool given any name and hash of file contents
-	Path(filename string, hashMD5 string) (string, error)
-	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
-	RelativePath(filename string, hashMD5 string) (string, error)
+	// Verify checks whether file exists in the pool and fills back checksum info
+	//
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
 	FilepathList(progress Progress) ([]string, error)
 	// Remove deletes file in package pool returns its size
 	Remove(path string) (size int64, err error)
-	// Import copies file into package pool
-	Import(path string, hashMD5 string) error
+}
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
 }
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
 type PublishedStorage interface {
-	// PublicPath returns root of public part
-	PublicPath() string
 	// MkDir creates directory recursively under public path
 	MkDir(path string) error
-	// CreateFile creates file for writing under public path
-	CreateFile(path string) (*os.File, error)
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
 	// RemoveDirs removes directory structure under public path
 	RemoveDirs(path string, progress Progress) error
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath string) error
+	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
-	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
+// FileSystemPublishedStorage is published storage on filesystem
+type FileSystemPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
+}
+
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -57,7 +138,7 @@ type Progress interface {
 	// Flush returns when all queued messages are sent
 	Flush()
 	// InitBar starts progressbar for count bytes or count items
-	InitBar(count int64, isBytes bool)
+	InitBar(count int64, isBytes bool, barType BarType)
 	// ShutdownBar stops progress bar and hides it
 	ShutdownBar()
 	// AddBar increments progress for progress bar
@@ -68,21 +149,29 @@ type Progress interface {
 	Printf(msg string, a ...interface{})
 	// ColoredPrintf does printf in colored way + newline
 	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
 }
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string, result chan<- error)
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
-	// Pause pauses task processing
-	Pause()
-	// Resume resumes task processing
-	Resume()
-	// Shutdown stops downloader after current tasks are finished,
-	// but doesn't process rest of queue
-	Shutdown()
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
+}
+
+// ChecksumStorageProvider creates ChecksumStorage based on DB
+type ChecksumStorageProvider func(db database.ReaderWriter) ChecksumStorage
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
 }

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
-// Version of aptly
-const Version = "0.5"
+// Version of aptly (filled in at link time)
+var Version string
 
-// Enable debugging features?
+// EnableDebug triggers some debugging features
 const EnableDebug = false

azure/azure.go

@@ -0,0 +1,2 @@
+// Package azure handles publishing to Azure Storage
+package azure

azure/azure_test.go

@@ -0,0 +1,12 @@
+package azure
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

azure/public.go

@@ -0,0 +1,375 @@
+package azure
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Azure)
+type PublishedStorage struct {
+	container azblob.ContainerURL
+	prefix    string
+	pathCache map[string]string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+func isEmulatorEndpoint(endpoint string) bool {
+	if h, _, err := net.SplitHostPort(endpoint); err == nil {
+		endpoint = h
+	}
+	if endpoint == "localhost" {
+		return true
+	}
+	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
+	// In this case, eliminate the '[' and ']' in the URL.
+	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
+	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
+		endpoint = endpoint[1 : len(endpoint)-1]
+	}
+	return net.ParseIP(endpoint) != nil
+}
+
+// NewPublishedStorage creates published storage from Azure storage credentials
+func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = "blob.core.windows.net"
+	}
+
+	var url *url.URL
+	if isEmulatorEndpoint(endpoint) {
+		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
+	} else {
+		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
+	result := &PublishedStorage{
+		container: containerURL,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for Azure
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+
+	sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename)
+	if err != nil {
+		return err
+	}
+
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(path, source, sourceMD5)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// putFile uploads file-like object to
+func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
+	path = filepath.Join(storage.prefix, path)
+
+	blob := storage.container.NewBlockBlobURL(path)
+
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
+	}
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
+		}
+	}
+
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob,
+		uploadOptions,
+	)
+
+	return err
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range filelist {
+		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+		if err != nil {
+			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
+		}
+	}
+
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
+	}
+	return err
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
+
+	relPath := filepath.Join(publishedDirectory, fileName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	if storage.pathCache == nil {
+		paths, md5s, err := storage.internalFilelist("")
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
+		}
+
+		storage.pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			storage.pathCache[paths[i]] = md5s[i]
+		}
+	}
+
+	destinationMD5, exists := storage.pathCache[relPath]
+	sourceMD5 := sourceChecksums.MD5
+
+	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+		}
+	}
+
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(relPath, source, sourceMD5)
+	if err == nil {
+		storage.pathCache[relPath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
+	}
+
+	return err
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := storage.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1000,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		}
+
+		marker = listBlob.NextMarker
+
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	paths, _, err := storage.internalFilelist(prefix)
+	return paths, err
+}
+
+// Internal copy or move implementation
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
+	const leaseDuration = 30
+
+	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
+	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
+	}
+	defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
+	srcBlobLeaseID := leaseResp.LeaseID()
+
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	copyStatus := copyResp.CopyStatus()
+	for {
+		if copyStatus == azblob.CopyStatusSuccess {
+			if move {
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
+				return err
+			}
+			return nil
+		} else if copyStatus == azblob.CopyStatusPending {
+			time.Sleep(1 * time.Second)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
+			if err != nil {
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
+			}
+			copyStatus = blobPropsResp.CopyStatus()
+
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
+			if err != nil {
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
+			}
+		} else {
+			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
+		}
+	}
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */)
+}
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		storageError, ok := err.(azblob.StorageError)
+		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+			return false, nil
+		}
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
+	}
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+	}
+	return resp.NewMetadata()["SymLink"], nil
+}

azure/public_test.go

@@ -0,0 +1,371 @@
+package azure
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type PublishedStorageSuite struct {
+	accountName, accountKey, endpoint string
+	storage, prefixedStorage          *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+const testContainerPrefix = "aptlytest-"
+
+func randContainer() string {
+	return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+	if n <= 0 {
+		panic("negative number")
+	}
+	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+	var bytes = make([]byte, n)
+	rand.Read(bytes)
+	for i, b := range bytes {
+		bytes[i] = alphanum[b%byte(len(alphanum))]
+	}
+	return string(bytes)
+}
+
+func (s *PublishedStorageSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+	cnt := s.storage.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	cnt := s.storage.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+	blob := s.storage.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, IsNil)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := ioutil.ReadAll(body)
+	c.Assert(err, IsNil)
+	return data
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+	_, err := s.storage.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, NotNil)
+	storageError, ok := err.(azblob.StorageError)
+	c.Assert(ok, Equals, true)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	hash := md5.Sum(data)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+
+	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b+c.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+	s.AssertNoFile(c, "a/b c.txt")
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestFilelistPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a 1", "a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a b", "a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	err = s.prefixedStorage.Remove("xyz")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlus(c *C) {
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.RenameFile("source.txt", "dest.txt")
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!"))
+
+	exists, err := s.storage.FileExists("source.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
+
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
+
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
+	c.Assert(err, IsNil)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
+	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in azure test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,109 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/aptly-dev/aptly/api"
+	"github.com/aptly-dev/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+		os.Remove(file)
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -2,35 +2,78 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"time"
+)
+
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	err = reflist.ForEach(func(key []byte) error {
-		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err2 != nil {
-			return err2
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	return PrintPackageList(list, "", "  ")
+
+}
+
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
+	if format == "" {
+		return result.ForEachIndexed(func(p *deb.Package) error {
+			context.Progress().Printf(prefix+"%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEachIndexed(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf(prefix+"%s\n", b.String())
+		return nil
+	})
+
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
 	return
 }
 
@@ -46,30 +89,37 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
+			makeCmdConfig(),
 			makeCmdDb(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
+			makeCmdAPI(),
 		},
 	}
 
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -1,281 +1,36 @@
 package cmd
 
 import (
-	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/console"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/http"
-	"github.com/smira/aptly/utils"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"runtime"
-	"runtime/pprof"
-	"strings"
-	"time"
 )
 
-// AptlyContext is a common context shared by all commands
-type AptlyContext struct {
-	flags        *flag.FlagSet
-	configLoaded bool
-
-	progress          aptly.Progress
-	downloader        aptly.Downloader
-	database          database.Storage
-	packagePool       aptly.PackagePool
-	publishedStorage  aptly.PublishedStorage
-	collectionFactory *deb.CollectionFactory
-	dependencyOptions int
-	architecturesList []string
-	// Debug features
-	fileCPUProfile *os.File
-	fileMemProfile *os.File
-	fileMemStats   *os.File
-}
-
-var context *AptlyContext
-
-// FatalError is type for panicking to abort execution with non-zero
-// exit code and print meaningful explanation
-type FatalError struct {
-	ReturnCode int
-	Message    string
-}
-
-// Fatal panics and aborts execution with exit code 1
-func Fatal(err error) {
-	panic(&FatalError{ReturnCode: 1, Message: err.Error()})
-}
-
-// Config loads and returns current configuration
-func (context *AptlyContext) Config() *utils.ConfigStructure {
-	if !context.configLoaded {
-		var err error
-
-		configLocation := context.flags.Lookup("config").Value.String()
-		if configLocation != "" {
-			err = utils.LoadConfig(configLocation, &utils.Config)
-
-			if err != nil {
-				Fatal(err)
-			}
-		} else {
-			configLocations := []string{
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-				"/etc/aptly.conf",
-			}
-
-			for _, configLocation := range configLocations {
-				err = utils.LoadConfig(configLocation, &utils.Config)
-				if err == nil {
-					break
-				}
-				if !os.IsNotExist(err) {
-					Fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-				}
-			}
-
-			if err != nil {
-				fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-				utils.SaveConfig(configLocations[0], &utils.Config)
-			}
-		}
-
-		context.configLoaded = true
-
-	}
-	return &utils.Config
-}
-
-// DependencyOptions calculates options related to dependecy handling
-func (context *AptlyContext) DependencyOptions() int {
-	if context.dependencyOptions == -1 {
-		context.dependencyOptions = 0
-		if context.Config().DepFollowSuggests || context.flags.Lookup("dep-follow-suggests").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowSuggests
-		}
-		if context.Config().DepFollowRecommends || context.flags.Lookup("dep-follow-recommends").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowRecommends
-		}
-		if context.Config().DepFollowAllVariants || context.flags.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowAllVariants
-		}
-		if context.Config().DepFollowSource || context.flags.Lookup("dep-follow-source").Value.Get().(bool) {
-			context.dependencyOptions |= deb.DepFollowSource
-		}
-	}
-
-	return context.dependencyOptions
-}
-
-// ArchitecturesList returns list of architectures fixed via command line or config
-func (context *AptlyContext) ArchitecturesList() []string {
-	if context.architecturesList == nil {
-		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.flags.Lookup("architectures").Value.String()
-		if optionArchitectures != "" {
-			context.architecturesList = strings.Split(optionArchitectures, ",")
-		}
-	}
-
-	return context.architecturesList
-}
-
-// Progress creates or returns Progress object
-func (context *AptlyContext) Progress() aptly.Progress {
-	if context.progress == nil {
-		context.progress = console.NewProgress()
-		context.progress.Start()
-	}
-
-	return context.progress
-}
-
-// Downloader returns instance of current downloader
-func (context *AptlyContext) Downloader() aptly.Downloader {
-	if context.downloader == nil {
-		context.downloader = http.NewDownloader(context.Config().DownloadConcurrency, context.Progress())
-	}
-
-	return context.downloader
-}
-
-// DBPath builds path to database
-func (context *AptlyContext) DBPath() string {
-	return filepath.Join(context.Config().RootDir, "db")
-}
-
-// Database opens and returns current instance of database
-func (context *AptlyContext) Database() (database.Storage, error) {
-	if context.database == nil {
-		var err error
-
-		context.database, err = database.OpenDB(context.DBPath())
-		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
-		}
-	}
-
-	return context.database, nil
-}
-
-// CollectionFactory builds factory producing all kinds of collections
-func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
-	if context.collectionFactory == nil {
-		db, err := context.Database()
-		if err != nil {
-			Fatal(err)
-		}
-		context.collectionFactory = deb.NewCollectionFactory(db)
-	}
-
-	return context.collectionFactory
-}
-
-// PackagePool returns instance of PackagePool
-func (context *AptlyContext) PackagePool() aptly.PackagePool {
-	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.Config().RootDir)
-	}
-
-	return context.packagePool
-}
-
-// PublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) PublishedStorage() aptly.PublishedStorage {
-	if context.publishedStorage == nil {
-		context.publishedStorage = files.NewPublishedStorage(context.Config().RootDir)
-	}
-
-	return context.publishedStorage
-}
+var context *ctx.AptlyContext
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if aptly.EnableDebug {
-		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-		if context.fileCPUProfile != nil {
-			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
-			context.fileCPUProfile = nil
-		}
-		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
-			context.fileMemProfile = nil
-		}
-	}
-	if context.database != nil {
-		context.database.Close()
-	}
-	if context.downloader != nil {
-		context.downloader.Shutdown()
-	}
-	if context.progress != nil {
-		context.progress.Shutdown()
-	}
+	context.Shutdown()
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	context.Cleanup()
 }
 
 // InitContext initializes context with default settings
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
-	context = &AptlyContext{flags: flags, dependencyOptions: -1}
-
-	if aptly.EnableDebug {
-		cpuprofile := flags.Lookup("cpuprofile").Value.String()
-		if cpuprofile != "" {
-			context.fileCPUProfile, err = os.Create(cpuprofile)
-			if err != nil {
-				return err
-			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
-		}
-
-		memprofile := flags.Lookup("memprofile").Value.String()
-		if memprofile != "" {
-			context.fileMemProfile, err = os.Create(memprofile)
-			if err != nil {
-				return err
-			}
-		}
-
-		memstats := flags.Lookup("memstats").Value.String()
-		if memstats != "" {
-			interval := flags.Lookup("meminterval").Value.Get().(time.Duration)
-
-			context.fileMemStats, err = os.Create(memstats)
-			if err != nil {
-				return err
-			}
-
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
-
-			go func() {
-				var stats runtime.MemStats
-
-				start := time.Now().UnixNano()
-
-				for {
-					runtime.ReadMemStats(&stats)
-					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
-							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
-						time.Sleep(interval)
-					} else {
-						break
-					}
-				}
-			}()
-		}
+	if context != nil {
+		panic("context already initialized")
 	}
 
-	return nil
+	context, err = ctx.NewContext(flags)
+
+	return err
+}
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
 }

cmd/db_cleanup.go

@@ -2,10 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
 	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
 )
 
 // aptly db cleanup
@@ -14,20 +17,106 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos and snapshots...\n")
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
 		}
 		return nil
 	})
@@ -35,13 +124,33 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
 		}
-		if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+		if published.SourceKind != deb.SourceLocalRepo {
+			return nil
+		}
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+		if e != nil {
+			return e
+		}
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 		return nil
 	})
@@ -49,51 +158,70 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
-		return nil
-	})
-	if err != nil {
-		return err
-	}
+	collectionFactory.Flush()
 
 	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
-	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
+	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Substract(existingPackageRefs)
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
+
+	if toDelete.Len() > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			batch := db.CreateBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+			})
+			if err != nil {
+				return fmt.Errorf("unable to delete by key: %s", err)
+			}
+
+			err = batch.Write()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	err = db.FinishBatch()
-	if err != nil {
-		return fmt.Errorf("unable to write to DB: %s", err)
-	}
+	collectionFactory.Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -112,7 +240,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -122,28 +250,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
-
-		var size, totalSize int64
-		for _, file := range filesToDelete {
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
+			for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
-	err = db.CompactDB()
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -163,5 +306,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/db_recover.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/database"
 	"github.com/smira/commander"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
@@ -11,11 +12,11 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = database.RecoverDB(context.DBPath())
+	err = goleveldb.RecoverDB(context.DBPath())
 
 	return err
 }

cmd/graph.go

@@ -2,136 +2,37 @@ package cmd
 
 import (
 	"bytes"
-	"code.google.com/p/gographviz"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strings"
-)
+	"time"
 
-func graphvizEscape(s string) string {
-	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
-}
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+)
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
 	var err error
 
-	graph := gographviz.NewGraph()
-	graph.SetDir(true)
-	graph.SetName("aptly")
-
-	existingNodes := map[string]bool{}
-
-	fmt.Printf("Loading mirrors...\n")
-
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkgoldenrod1",
-			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
-				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
-				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
 	}
 
-	fmt.Printf("Loading local repos...\n")
-
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
-		}
-
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "mediumseagreen",
-			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
-				repo.Name, repo.Comment, repo.NumPackages())),
-		})
-		existingNodes[repo.UUID] = true
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading snapshots...\n")
-
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		existingNodes[snapshot.UUID] = true
-		return nil
-	})
-
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
-		}
-
-		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
-			description = "Snapshot from repo"
-		}
-
-		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "cadetblue1",
-			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
-		})
-
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
-			for _, uuid := range snapshot.SourceIDs {
-				_, exists := existingNodes[uuid]
-				if exists {
-					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
-				}
-			}
-		}
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("Loading published repos...\n")
-
-	context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
-			"shape":     "Mrecord",
-			"style":     "filled",
-			"fillcolor": "darkolivegreen1",
-			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
-		})
-
-		_, exists := existingNodes[repo.SourceUUID]
-		if exists {
-			graph.AddEdge(graphvizEscape(repo.SourceUUID), "", graphvizEscape(repo.UUID), "", true, nil)
-		}
-
-		return nil
-	})
+	layout := context.Flags().Lookup("layout").Value.String()
 
 	fmt.Printf("Generating graph...\n")
+	collectionFactory := context.NewCollectionFactory()
+	graph, err := deb.BuildGraph(collectionFactory, layout)
+	if err != nil {
+		return err
+	}
 
 	buf := bytes.NewBufferString(graph.String())
 
@@ -142,9 +43,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -172,15 +80,51 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	defer func() {
+		_ = os.Remove(tempfilename)
+	}()
+
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		if err = viewer.Start(); err == nil {
+			// Wait for a second so that the visualizer has a chance to
+			// open the input file. This needs to be done even if we're
+			// waiting for the visualizer as it can be just a wrapper that
+			// spawns a browser tab and returns right away.
+			defer func(t <-chan time.Time) {
+				<-t
+			}(time.After(time.Second))
+		}
 	}
 
 	return err
 }
 
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
 func makeCmdGraph() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyGraph,
@@ -197,5 +141,9 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
 	return cmd
 }

cmd/mirror.go

@@ -1,20 +1,21 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
-func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if context.Config().GpgDisableVerify || flags.Lookup("ignore-signatures").Value.Get().(bool) {
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
@@ -54,6 +55,9 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorShow(),
 			makeCmdMirrorDrop(),
 			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
 	}
 }

cmd/mirror_create.go

@@ -2,20 +2,24 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	var err error
 	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -32,12 +36,25 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	verifier, err := getVerifier(context.flags)
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -47,7 +64,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -63,7 +81,8 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature.
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
+line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
 
@@ -77,7 +96,14 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,19 +11,25 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -34,7 +41,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
+	err = collectionFactory.RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/mirror_edit.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fetchMirror := false
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
+		}
+	})
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
+
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit mirror settings",
+		Long: `
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -1,24 +1,38 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	if jsonFlag {
+		return aptlyMirrorListJSON(cmd, args)
+	}
+
+	return aptlyMirrorListTxt(cmd, args)
+}
+
+func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -28,6 +42,8 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {
@@ -49,6 +65,32 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdMirrorList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorList,
@@ -63,6 +105,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/mirror_rename.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -1,42 +1,74 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/utils"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorShowJSON(cmd, args)
+	}
+
+	return aptlyMirrorShowTxt(cmd, args)
+}
+
+func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	downloadSources := "no"
+	downloadSources := No
 	if repo.DownloadSources {
-		downloadSources = "yes"
+		downloadSources = Yes
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := No
+	if repo.DownloadUdebs {
+		downloadUdebs = Yes
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := No
+		if repo.FilterWithDeps {
+			filterWithDeps = Yes
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
 	if repo.LastDownloadDate.IsZero() {
 		fmt.Printf("Last update: never\n")
 	} else {
@@ -49,18 +81,61 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("%s: %s\n", k, repo.Meta[k])
 	}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList())
+			ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
 	return err
 }
 
+func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				repo.Packages = append(repo.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(repo.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdMirrorShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorShow,
@@ -76,6 +151,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
 
 	return cmd

cmd/mirror_update.go

@@ -2,6 +2,13 @@ package cmd
 
 import (
 	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,24 +17,33 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
 
-	verifier, err := getVerifier(context.flags)
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+
+	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
@@ -37,12 +53,190 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch)
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	defer func() {
+		// on any interruption, unlock the mirror
+		err = context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			collectionFactory.RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.GoContextHandleSignals()
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
+
+	downloadQueue := make(chan int)
+
+	var (
+		errors  []string
+		errLock sync.Mutex
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
+	}
+
+	go func() {
+		for idx := range queue {
+			select {
+			case downloadQueue <- idx:
+			case <-context.Done():
+				return
+			}
+		}
+		close(downloadQueue)
+	}()
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case idx, ok := <-downloadQueue:
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						context,
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreMismatch)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					task.Done = true
+				case <-context.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	// Wait for all download goroutines to finish
+	wg.Wait()
+
+	context.Progress().ShutdownBar()
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
+
+	for idx := range queue {
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(collectionFactory, context.Progress())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -68,8 +262,13 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	if len(args) == 1 {
+		q, err = query.Parse(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search [<package-query>]",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query.
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,144 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+
+	return err
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
+			for _, f := range p.Files() {
+				var path string
+				path, err = f.GetPoolPath(packagePool)
+				if err != nil {
+					return err
+				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p, collectionFactory)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matching query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -1,19 +1,21 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
-	if flags.Lookup("skip-signing").Value.Get().(bool) || context.Config().GpgDisableSign {
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
 
 	err := signer.Init()
 	if err != nil {
@@ -35,6 +37,7 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishSnapshot(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
+			makeCmdPublishShow(),
 		},
 	}
 }

cmd/publish_drop.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -9,18 +11,23 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context.PublishedStorage(), prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		collectionFactory, context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -33,11 +40,11 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 func makeCmdPublishDrop() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "remove published repository",
 		Long: `
-Command removes whatever has been published under specified <prefix> and
-<distribution> name.
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
 
 Example:
 
@@ -45,5 +52,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
 	return cmd
 }

cmd/publish_list.go

@@ -1,31 +1,48 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
+	"os"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishListJSON(cmd, args)
+	}
+
+	return aptlyPublishListTxt(cmd, args)
+}
+
+func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
 		}
 
 		if raw {
-			published = append(published, fmt.Sprintf("%s %s", repo.Prefix, repo.Distribution))
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
 		} else {
 			published = append(published, repo.String())
 		}
@@ -36,6 +53,8 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
+	context.CloseDatabase()
+
 	sort.Strings(published)
 
 	if raw {
@@ -58,6 +77,42 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		repos = append(repos, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GetPath() < repos[j].GetPath()
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdPublishList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishList,
@@ -72,6 +127,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/publish_repo.go

@@ -8,13 +8,19 @@ import (
 func makeCmdPublishRepo() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "repo <name> [<prefix>]",
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish local repository",
 		Long: `
 Command publishes current state of local repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
 It is not recommended to publish local repositories directly unless the
 repository is for testing purposes and changes happen frequently. For
 production usage please take snapshot of repository and publish it
@@ -27,13 +33,23 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_show.go

@@ -0,0 +1,126 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishShowJSON(cmd, args)
+	}
+
+	return aptlyPublishShowTxt(cmd, args)
+}
+
+func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for component, sourceID := range repo.Sources {
+		var name string
+		if repo.SourceKind == deb.SourceSnapshot {
+			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == deb.SourceLocalRepo {
+			source, e := collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,110 +2,198 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 1 || len(args) > 2 {
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	collectionFactory := context.NewCollectionFactory()
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
-
-	var prefix string
-	if len(args) == 2 {
-		prefix = args[1]
+	var param string
+	if len(args) == len(components)+1 {
+		param = args[len(components)]
+		args = args[0 : len(args)-1]
 	} else {
-		prefix = ""
+		param = ""
 	}
+	storage, prefix := deb.ParsePrefix(param)
 
 	var (
-		source  interface{}
+		sources = []interface{}{}
 		message string
 	)
 
-	if cmd.Name() == "snapshot" {
-		var snapshot *deb.Snapshot
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+	if cmd.Name() == "snapshot" { // nolint: goconst
+		var (
+			snapshot     *deb.Snapshot
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, snapshot)
+			parts = append(parts, snapshot.Name)
+
+			if snapshot.NumPackages() == 0 {
+				emptyWarning = true
+			}
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
 		}
 
-		source = snapshot
-		message = fmt.Sprintf("Snapshot %s", snapshot.Name)
-	} else if cmd.Name() == "repo" {
-		var localRepo *deb.LocalRepo
-		localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else if cmd.Name() == "repo" { // nolint: goconst
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = collectionFactory.LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
-		if err != nil {
-			return fmt.Errorf("unable to publish: %s", err)
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
 		}
 
-		source = localRepo
-		message = fmt.Sprintf("Local repo %s", localRepo.Name)
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
 	} else {
 		panic("unknown command")
 	}
 
-	component := context.flags.Lookup("component").Value.String()
-	distribution := context.flags.Lookup("distribution").Value.String()
+	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
 
-	published, err := deb.NewPublishedRepo(prefix, distribution, component, context.ArchitecturesList(), source, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
+	published.Label = context.Flags().Lookup("label").Value.String()
+	published.Suite = context.Flags().Lookup("suite").Value.String()
 
-	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	published.SkipBz2 = context.Config().SkipBz2Publishing
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
+	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	prefix, component, distribution = published.Prefix, published.Component, published.Distribution
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
 	if prefix == "." {
 		prefix = ""
 	} else if !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
 
-	context.Progress().Printf("\n%s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		message, context.PublishedStorage().PublicPath())
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
+	}
+
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
-	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
-		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
 	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
@@ -116,13 +204,19 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 func makeCmdPublishSnapshot() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSnapshotOrRepo,
-		UsageLine: "snapshot <name> [<prefix>]",
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
 		Short:     "publish snapshot",
 		Long: `
 Command publishes snapshot as Debian repository ready to be consumed
 by apt tools. Published repostiories appear under rootDir/public directory.
 Valid GPG key is required for publishing.
 
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
 Example:
 
     $ aptly publish snapshot wheezy-main
@@ -130,13 +224,23 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
-	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -2,80 +2,121 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) < 2 || len(args) > 3 {
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	var (
-		name     string
+		names    []string
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 3 {
-		prefix = args[1]
-		name = args[2]
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
 	} else {
-		name = args[1]
+		names = args[1:]
 	}
 
-	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to switch: %s", err)
-	}
-
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to switch: %s", err)
-	}
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "snapshot" {
+	if published.SourceKind != deb.SourceSnapshot {
 		return fmt.Errorf("unable to update: not a snapshot publish")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	published.UpdateSnapshot(snapshot)
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
 
-	signer, err := getSigner(context.flags)
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
+
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, published.Component,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -86,22 +127,40 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [<prefix>] <new-snapshot>",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
 		Short:     "update published repository by switching to new snapshot",
 		Long: `
-Command switches in-place published repository with new snapshot contents. All
-publishing parameters are preserved (architecture list, distribution, component).
+Command switches in-place published snapshots with new snapshot contents. All
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
 
 Example:
 
-    $ aptly publish update wheezy ppa wheezy-7.5
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -11,53 +12,75 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	distribution := args[0]
-	prefix := "."
+	param := "."
 
 	if len(args) == 2 {
-		prefix = args[1]
+		param = args[1]
 	}
+	storage, prefix := deb.ParsePrefix(param)
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByPrefixDistribution(prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "local" {
+	if published.SourceKind != deb.SourceLocalRepo {
 		return fmt.Errorf("unable to update: not a local repository publish")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	published.UpdateLocalRepo()
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
 
-	signer, err := getSigner(context.flags)
+	signer, err := getSigner(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG signer: %s", err)
 	}
 
-	err = published.Publish(context.PackagePool(), context.PublishedStorage(), context.CollectionFactory(), signer, context.Progress())
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, published.Component,
-		context.PublishedStorage(), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -68,7 +91,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
-		UsageLine: "update <distribution> [<prefix>]",
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
 		Short:     "update published local repository",
 		Long: `
 Command re-publishes (updates) published local repository. <distribution>
@@ -76,6 +99,9 @@ and <prefix> should be occupied with local repository published
 using command aptly publish repo. Update happens in-place with
 minimum possible downtime for published repository.
 
+For multiple component published repositories, all local repositories
+are updated.
+
 Example:
 
     $ aptly publish update wheezy ppa
@@ -85,7 +111,14 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/repo.go

@@ -19,6 +19,9 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoMove(),
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
+			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -2,180 +2,89 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	packageFiles := []string{}
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	for _, location := range args[1:] {
-		info, err2 := os.Stat(location)
-		if err2 != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err2)
-			continue
-		}
-		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
-				if err3 != nil {
-					return err3
-				}
-				if info.IsDir() {
-					return nil
-				}
+	var packageFiles, otherFiles, failedFiles []string
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
-					packageFiles = append(packageFiles, path)
-				}
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
-				return nil
-			})
-		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
-				packageFiles = append(packageFiles, location)
-			} else {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
-				continue
-			}
-		}
+	var processedFiles, failedFiles2 []string
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		collectionFactory.PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		collectionFactory.ChecksumCollection)
+	failedFiles = append(failedFiles, failedFiles2...)
+	if err != nil {
+		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
-	processedFiles := []string{}
-	sort.Strings(packageFiles)
-
-	for _, file := range packageFiles {
-		var (
-			stanza deb.Stanza
-			p      *deb.Package
-		)
-
-		candidateProcessedFiles := []string{}
-		isSourcePackage := strings.HasSuffix(file, ".dsc")
-
-		if isSourcePackage {
-			stanza, err = deb.GetControlFileFromDsc(file, verifier)
-
-			if err == nil {
-				stanza["Package"] = stanza["Source"]
-				delete(stanza, "Source")
-
-				p, err = deb.NewSourcePackageFromControlFile(stanza)
-			}
-		} else {
-			stanza, err = deb.GetControlFileFromDeb(file)
-			p = deb.NewPackageFromControlFile(stanza)
-		}
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
-			continue
-		}
-
-		var checksums utils.ChecksumInfo
-		checksums, err = utils.ChecksumsForFile(file)
-		if err != nil {
-			return err
-		}
-
-		if isSourcePackage {
-			p.UpdateFiles(append(p.Files(), deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
-		} else {
-			p.UpdateFiles([]deb.PackageFile{deb.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
-		}
-
-		err = context.PackagePool().Import(file, checksums.MD5)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
-			continue
-		}
-
-		candidateProcessedFiles = append(candidateProcessedFiles, file)
-
-		// go over all files, except for the last one (.dsc/.deb itself)
-		for _, f := range p.Files() {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
-				continue
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = context.PackagePool().Import(sourceFile, f.Checksums.MD5)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
-				break
-			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
-		}
-		if err != nil {
-			// some files haven't been imported
-			continue
-		}
-
-		err = context.CollectionFactory().PackageCollection().Update(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
-			continue
-		}
-
-		err = list.Add(p)
-		if err != nil {
-			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
-			continue
-		}
-
-		context.Progress().ColoredPrintf("@g[+]@| %s added@|", p)
-		processedFiles = append(processedFiles, candidateProcessedFiles...)
-	}
+	processedFiles = append(processedFiles, otherFiles...)
 
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
 
-	if context.flags.Lookup("remove-files").Value.Get().(bool) {
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}
 		}
 	}
 
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
 	return err
 }
 
@@ -185,8 +94,8 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
 patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
 to the database. Files would be imported to internal package pool. For source packages, all required files are
 added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
@@ -199,6 +108,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_copy.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoCopy() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
 		Short:     "copy packages between local repositories",
 		Long: `
-Command copy copies packages matching <package-spec> from local repo
+Command copy copies packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_create.go

@@ -2,23 +2,49 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) != 1 {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repo := deb.NewLocalRepo(args[0], context.flags.Lookup("comment").Value.String())
-	repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
-	repo.DefaultComponent = context.flags.Lookup("component").Value.String()
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
-	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}
@@ -30,16 +56,21 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 func makeCmdRepoCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoCreate,
-		UsageLine: "create <name>",
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
 		Short:     "create local repository",
 		Long: `
 Create local package repository. Repository would be empty when
 created, packages could be added from files, copied or moved from
 another local repository or imported from the mirror.
 
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
 Example:
 
   $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
 `,
 		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
 	}
@@ -47,6 +78,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,21 +11,22 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	published := collectionFactory.PublishedRepoCollection().ByLocalRepo(repo)
 	if len(published) > 0 {
 		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -34,9 +36,9 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: local repo is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -48,7 +50,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
+	err = collectionFactory.LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/repo_edit.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/AlekSi/pointer"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,32 +13,47 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.flags.Lookup("comment").Value.String() != "" {
-		repo.Comment = context.flags.Lookup("comment").Value.String()
+	var uploadersFile *string
+
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
+
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
-	if context.flags.Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.flags.Lookup("distribution").Value.String()
-	}
-
-	if context.flags.Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.flags.Lookup("component").Value.String()
-	}
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -50,7 +68,7 @@ func makeCmdRepoEdit() *commander.Command {
 		UsageLine: "edit <name>",
 		Short:     "edit properties of local repository",
 		Long: `
-Command edit allows to change metadata of local repository:
+Command edit allows one to change metadata of local repository:
 comment, default distribution and component.
 
 Example:
@@ -63,6 +81,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_import.go

@@ -8,10 +8,10 @@ import (
 func makeCmdRepoImport() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
 		Short:     "import packages from mirror to local repository",
 		Long: `
-Command import looks up packages matching <package-spec> in mirror <src-mirror>
+Command import looks up packages matching <package-query> in mirror <src-mirror>
 and copies them to local repo <dst-repo>.
 
 Example:

cmd/repo_include.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"fmt"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = context.GetVerifier()
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+	collectionFactory := context.NewCollectionFactory()
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, failedFiles2 []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
+		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+		context.PackagePool(), collectionFactory.ChecksumCollection,
+		uploaders, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -1,30 +1,44 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoListJSON(cmd, args)
+	}
+
+	return aptlyRepoListTxt(cmd, args)
+}
+
+func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-			if err != nil {
-				return err
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
@@ -33,6 +47,8 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
+	context.CloseDatabase()
+
 	sort.Strings(repos)
 
 	if raw {
@@ -55,6 +71,36 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdRepoList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoList,
@@ -69,6 +115,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/repo_move.go

@@ -2,27 +2,30 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 3 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	command := cmd.Name()
 
-	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -32,8 +35,8 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		srcRepo    *deb.LocalRepo
 	)
 
-	if command == "copy" || command == "move" {
-		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	if command == "copy" || command == "move" { // nolint: goconst
+		srcRepo, err = collectionFactory.LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -42,21 +45,21 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
 		srcRefList = srcRepo.RefList()
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+		srcRemoteRepo, err = collectionFactory.RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -72,12 +75,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -86,7 +89,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	var architecturesList []string
 
-	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
 
 	if withDeps {
 		dstList.PrepareIndex()
@@ -105,18 +108,26 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.DependencyOptions(), architecturesList)
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var verb string
 
-	if command == "move" {
+	if command == "move" { // nolint: goconst
 		verb = "moved"
-	} else if command == "copy" {
+	} else if command == "copy" { // nolint: goconst
 		verb = "copied"
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		verb = "imported"
 	}
 
@@ -126,7 +137,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcList.Remove(p)
 		}
 		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
@@ -136,20 +147,20 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -162,10 +173,10 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 func makeCmdRepoMove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoMoveCopyImport,
-		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
 		Short:     "move packages between local repositories",
 		Long: `
-Command move moves packages matching <package-spec> from local repo
+Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
 Example:

cmd/repo_remove.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -11,30 +13,39 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
 	list.PrepareIndex()
-	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -45,12 +56,12 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -62,10 +73,10 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 func makeCmdRepoRemove() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoRemove,
-		UsageLine: "remove <name> <package-spec> ...",
+		UsageLine: "remove <name> <package-query> ...",
 		Short:     "remove packages from local repository",
 		Long: `
-Commands removes packages matching <package-spec> from local repository
+Commands removes packages matching <package-query> from local repository
 <name>. If removed packages are not referenced by other repos or
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.

cmd/repo_rename.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = collectionFactory.LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/repo_show.go

@@ -1,26 +1,42 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoShowJSON(cmd, args)
+	}
+
+	return aptlyRepoShowTxt(cmd, args)
+}
+
+func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -29,11 +45,56 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList())
+		ListPackagesRefList(repo.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	packageList := []string{}
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err == nil {
+				packageList = list.FullNames()
+			}
+		}
+
+		sort.Strings(packageList)
+	}
+
+	// merge the repo object with the package list
+	var output []byte
+	if output, err = json.MarshalIndent(struct {
+		*deb.LocalRepo
+		Packages []string
+	}{repo, packageList}, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -53,6 +114,7 @@ ex:
 		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/run.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*ctx.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			ctx.Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -2,25 +2,45 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
 	var err error
 
-	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if collectionFactory.PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
 
-	listen := context.flags.Lookup("listen").Value.String()
+	listen := context.Flags().Lookup("listen").Value.String()
 
 	listenHost, listenPort, err := net.SplitHostPort(listen)
 
@@ -37,13 +57,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
+	sources := make(sort.StringSlice, 0, collectionFactory.PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		if e != nil {
+			return e
 		}
 
 		sources = append(sources, repo.String())
@@ -69,15 +89,15 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
-			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
-		if utils.StrSliceHasItem(repo.Architectures, "source") {
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
-				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.PublishedStorage().PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot.go

@@ -17,6 +17,9 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotDiff(),
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
 	}
 }

cmd/snapshot_create.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -12,18 +13,24 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
 		repoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -32,18 +39,18 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
-	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
 		// aptly snapshot create snap from repo repo
 		var repo *deb.LocalRepo
 
 		localRepoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -61,10 +68,10 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
 	} else {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}

cmd/snapshot_diff.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,35 +11,36 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	onlyMatching := context.flags.Lookup("only-matching").Value.Get().(bool)
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name-a> snapshot
-	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshotA, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	snapshotB, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -10,22 +11,23 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
+	published := collectionFactory.PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -35,9 +37,9 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: snapshot is published")
 	}
 
-	force := context.flags.Lookup("force").Value.Get().(bool)
+	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
+		snapshots := collectionFactory.SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -48,7 +50,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
+	err = collectionFactory.SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/snapshot_filter.go

@@ -0,0 +1,109 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <source> snapshot
+	source, err := collectionFactory.SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = collectionFactory.SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -1,46 +1,53 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
-	"sort"
 )
 
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlySnapshotListJSON(cmd, args)
+	}
+
+	return aptlySnapshotListTxt(cmd, args)
+}
+
+func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	snapshots := make([]string, context.CollectionFactory().SnapshotCollection().Len())
-
-	i := 0
-	context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		if raw {
-			snapshots[i] = snapshot.Name
-		} else {
-			snapshots[i] = snapshot.String()
-		}
-		i++
-		return nil
-	})
-
-	sort.Strings(snapshots)
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
-		for _, snapshot := range snapshots {
-			fmt.Printf("%s\n", snapshot)
-		}
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+			fmt.Printf("%s\n", snapshot.Name)
+			return nil
+		})
 	} else {
-		if len(snapshots) > 0 {
+		if collection.Len() > 0 {
 			fmt.Printf("List of snapshots:\n")
 
-			for _, snapshot := range snapshots {
-				fmt.Printf(" * %s\n", snapshot)
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
 			}
 
 			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
@@ -48,8 +55,31 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
 		}
 	}
-	return err
 
+	return err
+}
+
+func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	collection := context.NewCollectionFactory().SnapshotCollection()
+
+	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
+	i := 0
+	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		jsonSnapshots[i] = snapshot
+		i++
+		return nil
+	})
+	if output, e := json.MarshalIndent(jsonSnapshots, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
 }
 
 func makeCmdSnapshotList() *commander.Command {
@@ -66,7 +96,9 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
 
 	return cmd
 }

cmd/snapshot_merge.go

@@ -2,43 +2,50 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 2 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
+	collectionFactory := context.NewCollectionFactory()
 	sources := make([]*deb.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
+		sources[i], err = collectionFactory.SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 	}
 
-	latest := context.flags.Lookup("latest").Value.Get().(bool)
-	overrideMatching := !latest
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
+
+	if noRemove && latest {
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
+	}
+
+	overrideMatching := !latest && !noRemove
 
 	result := sources[0].RefList()
-
 	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList(), overrideMatching)
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
 	}
 
 	if latest {
-		deb.FilterLatestRefs(result)
+		result.FilterLatestRefs()
 	}
 
 	sourceDescription := make([]string, len(sources))
@@ -50,7 +57,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -79,6 +86,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
 
 	return cmd
 }

cmd/snapshot_pull.go

@@ -2,41 +2,45 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"sort"
 	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 4 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	noDeps := context.flags.Lookup("no-deps").Value.Get().(bool)
-	noRemove := context.flags.Lookup("no-remove").Value.Get().(bool)
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name> snapshot
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	source, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -46,12 +50,12 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	// Convert snapshot to package list
 	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -75,86 +79,66 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
 	}
 
-	// Initial dependencies out of arguments
-	initialDependencies := make([]deb.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = deb.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+	for _, arch := range architecturesList {
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
 	}
 
-	// Perform pull
-	for _, arch := range architecturesList {
-		dependencies := make([]deb.Dependency, len(initialDependencies), 128)
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
 		}
 
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				continue
-			}
-
-			if !noRemove {
-				// Remove all packages with the same name and architecture
-				for p := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
-					packageList.Remove(p)
-					context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
-					p = packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
-				}
-			}
-
-			// Add new discovered package
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
 			packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := deb.NewPackageList()
-			pL.Add(pkg)
-
-			var missing []deb.Dependency
-			missing, err = pL.VerifyDependencies(context.DependencyOptions(), []string{arch}, packageList, nil)
-			if err != nil {
-				context.Progress().ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
 		}
-	}
 
-	if context.flags.Lookup("dry-run").Value.Get().(bool) {
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
+
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
 	} else {
 		// Create <destination> snapshot
 		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = context.CollectionFactory().SnapshotCollection().Add(destination)
+		err = collectionFactory.SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -167,14 +151,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotPull() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
 		Short:     "pull packages from another snapshot",
 		Long: `
 Command pull pulls new packages along with its' dependencies to snapshot <name>
 from snapshot <source>. Pull can upgrade package version in <name> with
 versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
-as 'package-name' or as dependency 'package-name (>= version)'.
+as 'package-name' or as package queries.
 
 Example:
 
@@ -186,6 +170,7 @@ Example:
 	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
 	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
 	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
 
 	return cmd
 }

cmd/snapshot_rename.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err = collectionFactory.SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_search.go

@@ -0,0 +1,143 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+	collectionFactory := context.NewCollectionFactory()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" { // nolint: goconst
+		var snapshot *deb.Snapshot
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		var repo *deb.RemoteRepo
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" { // nolint: goconst
+		var repo *deb.LocalRepo
+		repo, err = collectionFactory.LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	if len(args) == 2 {
+		q, err = query.Parse(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -1,26 +1,41 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlySnapshotShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
-	name := args[0]
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	if jsonFlag {
+		return aptlySnapshotShowJSON(cmd, args)
+	}
+
+	return aptlySnapshotShowTxt(cmd, args)
+}
+
+func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -29,10 +44,113 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
 	fmt.Printf("Description: %s\n", snapshot.Description)
 	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+	if len(snapshot.SourceIDs) > 0 {
+		fmt.Printf("Sources:\n")
+		for _, sourceID := range snapshot.SourceIDs {
+			var name string
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = collectionFactory.SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = collectionFactory.RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			}
 
-	withPackages := context.flags.Lookup("with-packages").Value.Get().(bool)
+			if name != "" {
+				fmt.Printf("  %s [%s]\n", name, snapshot.SourceKind)
+			}
+		}
+	}
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(snapshot.RefList())
+		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	snapshot, err := context.NewCollectionFactory().SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include the sources
+	if len(snapshot.SourceIDs) > 0 {
+		for _, sourceID := range snapshot.SourceIDs {
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = context.NewCollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.Snapshots = append(snapshot.Snapshots, source)
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.LocalRepos = append(snapshot.LocalRepos, source)
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = context.NewCollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.RemoteRepos = append(snapshot.RemoteRepos, source)
+			}
+		}
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if snapshot.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(snapshot.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(snapshot, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -53,6 +171,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/snapshot_verify.go

@@ -2,26 +2,28 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	var err error
 	if len(args) < 1 {
 		cmd.Usage()
-		return err
+		return commander.ErrCommandError
 	}
 
 	snapshots := make([]*deb.Snapshot, len(args))
+	collectionFactory := context.NewCollectionFactory()
 	for i := range snapshots {
-		snapshots[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i])
+		snapshots[i], err = collectionFactory.SnapshotCollection().ByName(args[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshots[i])
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
@@ -29,27 +31,27 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
+		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	sourcePackageList := deb.NewPackageList()
 	err = sourcePackageList.Append(packageList)
 	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
+		return fmt.Errorf("unable to merge sources: %s", err)
 	}
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
-		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), collectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
+			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		err = sourcePackageList.Append(pL)
 		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
+			return fmt.Errorf("unable to merge sources: %s", err)
 		}
 	}
 

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s", filename)
+		}
+
+		fmt.Print("Reading file...\n\n")
+
+		var file *os.File
+		file, err = os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			err = context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+	  $ aptly task run
+	  > repo create local
+	  > repo add local pkg1
+	  > publish repo local
+	  > serve
+	  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}