S3: support disabling ACL with none value

This change lets you disable ACL when using S3 by using a configuration
value of `none`. This way we maintain backward compatibility with the
default setting being `private`.

Fixes: #1067

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 200
+ignore = E126,E241,E741,W504
+include =
+    system
+exclude =
+    system/env

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
+
+name: CI
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  build:
+    name: test
+    runs-on: ubuntu-20.04
+    continue-on-error: ${{ matrix.allow_failure }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+      matrix:
+        go: [1.16, 1.17, 1.18]
+        run_long_tests: [no]
+        allow_failure: [false]
+        include:
+          # Disable this for now as it's not clear how to select the latest master branch
+          # version of go using actions/setup-go@v2.
+          # - go: master
+          #   run_long_tests: no
+          #   allow_failure: true
+          - go: 1.17
+            run_long_tests: yes
+            allow_failure: false
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.45.0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install O/S packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+
+      - name: Install Python packages
+        run: |
+          pip install six packaging appdirs virtualenv
+          pip install -U pip setuptools
+          pip install -r system/requirements.txt
+
+      - name: Install Azurite
+        id: azuright
+        uses: potatoqualitee/azuright@v1.1
+        with:
+          directory: ${{ runner.temp }}
+
+      - name: Get aptly version
+        run: |
+           make version
+
+      - name: Make
+        env:
+          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
+          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        run: |
+          make
+
+      - name: Upload code coverage
+        if: matrix.run_long_tests
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.txt
+
+  release:
+    name: release
+    needs: build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # fetch the whole repot for `git describe` to
+          # work and get the nightly verion
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+
+      - name: Make Release
+        env:
+          GOBIN: /usr/local/bin
+        run: |
+          make release
+
+      - name: Publish nightly release to aptly
+        if: github.ref == 'refs/heads/master'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh nightly
+
+      - name: Publish release to aptly
+        if: startsWith(github.event.ref, 'refs/tags')
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh release
+
+      - name: Upload artifacts to GitHub Release
+        if: startsWith(github.event.ref, 'refs/tags')
+        uses: softprops/action-gh-release@v1
+        with:
+          body: Release ${{ github.ref }} generated by the CI.
+          files: build/*

.gitignore

@@ -22,7 +22,23 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
+coverage.txt
-coverage*.out
 
 *.pyc
+
+xc-out/
+root/
+
+man/aptly.1.html
+man/aptly.1.ronn
+
+.goxc.local.json
+
+system/env/
+
+# created by make build for release artifacts
+build/
+
+pgp/keyrings/aptly2*.gpg
+pgp/keyrings/aptly2*.gpg~
+pgp/keyrings/.#*

.golangci.yml

@@ -0,0 +1,19 @@
+run:
+  tests: false
+
+
+linters:
+  disable-all: true
+  enable:
+   - deadcode
+   - goconst
+   - gofmt
+   - goimports
+   - govet
+   - ineffassign
+   - misspell
+   - revive
+   - staticcheck
+   - structcheck
+   - varcheck
+   - vetshadow

.goxc.json

@@ -0,0 +1,45 @@
+{
+	"AppName": "aptly",
+	"ArtifactsDest": "xc-out/",
+	"TasksExclude": [
+		"rmbin",
+		"go-test",
+		"go-vet"
+	],
+    "TasksAppend": [
+		"bintray"
+    ],
+    "TaskSettings": {
+        "debs": {
+            "metadata": {
+                "maintainer": "Andrey Smirnov",
+                "maintainer-email": "me@smira.ru",
+                "description": "Debian repository management tool"
+            },
+            "metadata-deb": {
+                "License": "MIT",
+                "Homepage": "https://www.aptly.info/",
+                "Depends": "bzip2, xz-utils, gnupg, gpgv",
+				"Suggests": "graphviz"
+            },
+            "other-mapped-files": {
+                "/": "root/"
+            }
+        },
+        "bintray": {
+            "repository": "aptly",
+            "subject": "smira",
+            "package": "aptly",
+            "downloadspage": "bintray.md"
+        }
+    },
+	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
+	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
+	"MainDirsExclude": "_man,vendor",
+	"BuildSettings": {
+		"LdFlagsXVars": {
+			"Version": "main.Version"
+		}
+	},
+	"ConfigVersion": "0.9"
+}

.travis.yml

@@ -1,15 +1,106 @@
+dist: xenial
+sudo: required
+
 language: go
 
-go:
+go_import_path: github.com/aptly-dev/aptly
-  - 1.1
+
-  - 1.2
+addons:
+  apt:
+    packages:
+    - python-virtualenv
+    - graphviz
+    - gnupg2
+    - gpgv2
 
 env:
   global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
+    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
+    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+    - NO_FTP_ACCESS: "yes"
+    - BOTO_CONFIG: /dev/null
+    - GO111MODULE: "on"
+    - GOPROXY: https://proxy.golang.org
+
+matrix:
+  allow_failures:
+    - go: master
+      env: RUN_LONG_TESTS=no
+  fast_finish: true
+  include:
+    - go: 1.11.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.12.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.13.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.14.x
+      env: RUN_LONG_TESTS=yes
+    - go: 1.15.x
+      env:
+        - RUN_LONG_TESTS=yes
+        - DEPLOY_BINARIES=yes
+        - APTLY_USER=aptly
+        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
+    - go: master
+      env: RUN_LONG_TESTS=no
+
+before_install:
+  - virtualenv system/env
+  - . system/env/bin/activate
+  - pip install six packaging appdirs
+  - pip install -U pip setuptools
+  - pip install -r system/requirements.txt
+  - make version
 
 install:
   - make prepare
 
+after_success:
+  - bash <(curl -s https://codecov.io/bash)
 
-script: make travis
+notifications:
+  webhooks:
+    urls:
+      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
+    on_success: change  # options: [always|never|change] default: always
+    on_failure: always  # options: [always|never|change] default: always
+    on_start: false     # default: false
+
+before_deploy:
+  - make release
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
+    file_glob: true
+    file: build/*
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: s3
+    access_key_id:
+      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
+    secret_access_key:
+      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
+    bucket: aptly-nightly
+    skip_cleanup: true
+    acl: public-read
+    local_dir: build
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh nightly
+    skip_cleanup: true
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh release
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -0,0 +1,51 @@
+List of contributors, in chronological order:
+
+* Andrey Smirnov (https://github.com/smira)
+* Sebastien Binet (https://github.com/sbinet)
+* Ryan Uber (https://github.com/ryanuber)
+* Simon Aquino (https://github.com/queeno)
+* Vincent Batoufflet (https://github.com/vbatoufflet)
+* Ivan Kurnosov (https://github.com/zerkms)
+* Dmitrii Kashin (https://github.com/freehck)
+* Chris Read (https://github.com/cread)
+* Rohan Garg (https://github.com/shadeslayer)
+* Russ Allbery (https://github.com/rra)
+* Sylvain Baubeau (https://github.com/lebauce)
+* Andrea Bernardo Ciddio (https://github.com/bcandrea)
+* Michael Koval (https://github.com/mkoval)
+* Alexander Guy (https://github.com/alexanderguy)
+* Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)
+* Shengjing Zhu (https://github.com/zhsj)
+* Nabil Bendafi (https://github.com/nabilbendafi)
+* Raphael Medaer (https://github.com/rmedaer)
+* Raul Benencia (https://github.com/rul)
+* Don Kuntz (https://github.com/dkuntz2)
+* Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)
+* Markus Muellner (https://github.com/mmianl)
+* Chuan Liu (https://github.com/chuan)
+* Samuel Mutel (https://github.com/smutel)
+* Russell Greene (https://github.com/russelltg)
+* Wade Simmons (https://github.com/wadey)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -0,0 +1,241 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+
+### List of Repositories
+
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly.info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Your First Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+## Development Setup
+
+This section describes local setup to start contributing to aptly source.
+
+### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.
+
+### Forking and Cloning
+
+As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
+
+    git clone git@github.com:aptly-dev/aptly.git
+    cd aptly
+
+For main repo under your GitHub user and add it as another Git remote:
+
+    git remote add <user> git@github.com:<user>/aptly.git
+
+That way you can continue to build project as is (you don't need to adjust import paths), but you would need
+to specify your remote name when pushing branches:
+
+    git push <user> <your-branch>
+
+### Dependencies
+
+You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+
+    make prepare dev system/env
+
+This is usually one-time action.
+
+Aptly is using Go modules to manage dependencies, download modules using:
+
+    make modules
+
+### Building
+
+If you want to build aptly binary from your current source tree, run:
+
+    make install
+
+This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
+
+    aptly
+
+Or, if it's not on your path:
+
+    ~/go/bin/aptly
+
+### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### Style Checks
+
+Style checks could be run with:
+
+    make check
+
+aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
+for the linter could be found in [.golangci.yml](.golangci.yml) file.
+
+Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
+
+### Vendored Code
+
+aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
+repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
+packages over packages in `$GOPATH`.
+
+If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
+Usually all you need is `dep ensure` or `dep ensure -update`.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash and Zsh Completion
+
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.
+
+## Design
+
+This section requires future work.
+
+*TBD*
+
+### Database
+
+### Package Pool
+
+### Package
+
+### PackageList, PackageRefList
+
+### LocalRepo, RemoteRepo, Snapshot
+
+### PublishedRepository
+
+### Context
+
+### Collections, CollectionFactory

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,60 +1,81 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-
+ifdef TRAVIS_TAG
-ifeq ($(TRAVIS), true)
+	TAG=$(TRAVIS_TAG)
-GOVERALLS?=$(HOME)/gopath/bin/goveralls
-SRCPATH?=$(HOME)/gopath/src
-BINPATH=$(HOME)/gopath/bin
 else
-GOVERALLS?=goveralls
+	TAG="$(shell git describe --tags --always)"
-SRCPATH?=$(GOPATH)/src
+endif
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
+PACKAGES=context database deb files gpg http query swift s3 utils
+PYTHON?=python3
+TESTS?=
 BINPATH?=$(GOPATH)/bin
+RUN_LONG_TESTS?=yes
+
+all: modules test bench check system-test
+
+prepare:
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+
+modules:
+	go mod download
+	go mod verify
+	go mod tidy -v
+
+dev:
+	go get -u github.com/laher/goxc
+
+check: system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	golangci-lint run
+	system/env/bin/flake8
 endif
 
-ifeq ($(GOVERSION), go1.2)
+install:
-TRAVIS_TARGET=coveralls
+	go install -v -ldflags "-X main.Version=$(VERSION)"
-PREPARE_LIST=cover-prepare
+
-else
+system/env: system/requirements.txt
-TRAVIS_TARGET=test
+ifeq ($(RUN_LONG_TESTS), yes)
-PREPARE_LIST=
+	rm -rf system/env
+	$(PYTHON) -m venv system/env
+	system/env/bin/pip install -r system/requirements.txt
 endif
 
-all: test check system-test
+system-test: install system/env
-
+ifeq ($(RUN_LONG_TESTS), yes)
-prepare: $(PREPARE_LIST)
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-	go get -d -v ./...
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	go get launchpad.net/gocheck
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
-
+endif
-cover-prepare:
-	go get github.com/golang/lint/golint
-	go get github.com/mattn/goveralls
-	go get github.com/axw/gocov/gocov
-	go get code.google.com/p/go.tools/cmd/cover
-
-coverage.out:
-	go test -coverprofile=coverage.debian.out -covermode=count ./debian
-	go test -coverprofile=coverage.utils.out -covermode=count ./utils
-	go test -coverprofile=coverage.database.out -covermode=count ./database
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-
-coverage: coverage.out
-	go tool cover -html=coverage.out
-	rm -f coverage.out
-
-check:
-	go tool vet -all=true .
-	golint .
-
-system-test:
-	go install
-	PATH=$(BINPATH):$(PATH) python system/run.py
-
-travis: $(TRAVIS_TARGET) system-test
 
 test:
-	go test -v ./... -gocheck.v=true
+	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
 
-coveralls: coverage.out
+bench:
-	@$(GOVERALLS) -service travis-ci.org -coverprofile=coverage.out $(COVERALLS_TOKEN)
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
-.PHONY: coverage.out
+mem.png: mem.dat mem.gp
+	gnuplot mem.gp
+	open mem.png
+
+goxc: dev
+	rm -rf root/
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
+	cp man/aptly.1 root/usr/share/man/man1
+	cp completion.d/aptly root/etc/bash_completion.d/
+	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
+	gzip root/usr/share/man/man1/aptly.1
+	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
+
+release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
+release: goxc
+	rm -rf build/
+	mkdir -p build/
+	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
+
+man:
+	make -C man
+
+version:
+	@echo $(VERSION)
+
+.PHONY: man modules version release goxc

README.rst

@@ -2,15 +2,24 @@
 aptly
 =====
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
+.. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
-    :target: https://travis-ci.org/smira/aptly
+    :target: https://github.com/aptly-dev/aptly/actions
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+    :target: https://codecov.io/gh/aptly-dev/aptly
+
+.. image:: https://badges.gitter.im/Join Chat.svg
+    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+
+.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
+    :target: https://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
 Aptly features: ("+" means planned features)
@@ -19,24 +28,90 @@ Aptly features: ("+" means planned features)
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
-* merge two or more snapshots into one (+)
+* merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
+* filter repository by search query, pulling dependencies when required
-* publish self-made packages as Debian repositories (+)
+* publish self-made packages as Debian repositories
+* REST API for remote access
+* mirror repositories "as-is" (without resigning with user's key) (+)
+* support for yum repositories (+)
 
 Current limitations:
 
-* source packages, debian-installer and translations not supported yet
+* translations are not supported yet
-* checksums and signature are not verified while downloading
-* deleting created items is not implemented
-* cleaning up stale files is not implemented
-
-Currently aptly is under heavy development, so please use it with care.
 
 Download
 --------
 
-Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
-If you have Go environment set up, you can build aptly from source by running::
+    deb http://repo.aptly.info/ squeeze main
 
-    go get github.com/smira/aptly
+And import key that is used to sign the release::
+
+    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
+
+After that you can install aptly as any other software package::
+
+    $ apt-get update
+    $ apt-get install aptly
+
+Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
+Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+
+If you would like to use nightly builds (unstable), please use following repository::
+
+    deb http://repo.aptly.info/ nightly main
+
+Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
+
+If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
+
+    git clone https://github.com/aptly-dev/aptly
+    cd aptly
+    make modules install
+
+Binary would be installed to ``$GOPATH/bin/aptly``.
+
+Contributing
+------------
+
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+
+Integrations
+------------
+
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
+
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func allFlags(flags *flag.FlagSet) []*flag.Flag {
+	result := []*flag.Flag{}
+	flags.VisitAll(func(f *flag.Flag) {
+		result = append(result, f)
+	})
+	return result
+}
+
+func findCommand(cmd *commander.Command, name string) (*commander.Command, error) {
+	for _, c := range cmd.Subcommands {
+		if c.Name() == name {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("command %s not found", name)
+}
+
+func capitalize(s string) string {
+	parts := strings.Split(s, " ")
+	for i, part := range parts {
+		if part[0] != '<' && part[0] != '[' && part[len(part)-1] != '>' && part[len(part)-1] != ']' {
+			parts[i] = "`" + part + "`"
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
+func main() {
+	command := cmd.RootCommand()
+	command.UsageLine = "aptly"
+	command.Dispatch(nil)
+
+	_File, _ := filepath.Abs("./man")
+
+	templ := template.New("man").Funcs(template.FuncMap{
+		"allFlags":    allFlags,
+		"findCommand": findCommand,
+		"toUpper":     strings.ToUpper,
+		"capitalize":  capitalize,
+		"authors":     authors,
+	})
+	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
+
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
+	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = templ.ExecuteTemplate(output, "main", command)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	output.Close()
+
+	out, err := exec.Command("ronn", filepath.Join(filepath.Dir(_File), "aptly.1.ronn")).CombinedOutput()
+	if err != nil {
+		os.Stdout.Write(out)
+		log.Fatal(err)
+	}
+
+	cmd := exec.Command("man", filepath.Join(filepath.Dir(_File), "aptly.1"))
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

api/api.go

@@ -0,0 +1,221 @@
+// Package api provides implementation of aptly REST API
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// Lock order acquisition (canonical):
+//  1. RemoteRepoCollection
+//  2. LocalRepoCollection
+//  3. SnapshotCollection
+//  4. PublishedRepoCollection
+
+// GET /api/version
+func apiVersion(c *gin.Context) {
+	c.JSON(200, gin.H{"Version": aptly.Version})
+}
+
+type dbRequestKind int
+
+const (
+	acquiredb dbRequestKind = iota
+	releasedb
+)
+
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
+var dbRequests chan dbRequest
+
+// Acquire database lock and release it when not needed anymore.
+//
+// Should be run in a goroutine!
+func acquireDatabase() {
+	clients := 0
+	for request := range dbRequests {
+		var err error
+
+		switch request.kind {
+		case acquiredb:
+			if clients == 0 {
+				err = context.ReOpenDatabase()
+			}
+
+			request.err <- err
+
+			if err == nil {
+				clients++
+			}
+		case releasedb:
+			clients--
+			if clients == 0 {
+				err = context.CloseDatabase()
+			} else {
+				err = nil
+			}
+
+			request.err <- err
+		}
+	}
+}
+
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Println("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			c.AbortWithError(409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Println("Executing task synchronously")
+		out := context.Progress()
+		detail := task.Detail{}
+		retValue, err := proc(out, &detail)
+		if err != nil {
+			c.AbortWithError(retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
+	}
+}
+
+// Common piece of code to show list of packages,
+// with searching & details if requested
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			return
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}

api/api_test.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"encoding/json"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ApiSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+}
+
+var _ = Suite(&ApiSuite{})
+
+func createTestConfig() *os.File {
+	file, err := ioutil.TempFile("", "aptly")
+	if err != nil {
+		return nil
+	}
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+	})
+	if err != nil {
+		return nil
+	}
+	file.Write(jsonString)
+	return file
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	file := createTestConfig()
+	c.Assert(file, NotNil)
+	s.configFile = file
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *ApiSuite) TearDownSuite(c *C) {
+	os.Remove(s.configFile.Name())
+	s.context.Shutdown()
+}
+
+func (s *ApiSuite) SetUpTest(c *C) {
+}
+
+func (s *ApiSuite) TearDownTest(c *C) {
+}
+
+func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w, nil
+}
+
+func (s *ApiSuite) TestGetVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/version", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, ".*Version.*")
+}
+
+func (s *ApiSuite) TestTruthy(c *C) {
+	c.Check(truthy("no"), Equals, false)
+	c.Check(truthy("n"), Equals, false)
+	c.Check(truthy("off"), Equals, false)
+	c.Check(truthy("false"), Equals, false)
+	c.Check(truthy("0"), Equals, false)
+	c.Check(truthy(false), Equals, false)
+	c.Check(truthy(0), Equals, false)
+
+	c.Check(truthy("y"), Equals, true)
+	c.Check(truthy("yes"), Equals, true)
+	c.Check(truthy("t"), Equals, true)
+	c.Check(truthy("true"), Equals, true)
+	c.Check(truthy("1"), Equals, true)
+	c.Check(truthy(true), Equals, true)
+	c.Check(truthy(1), Equals, true)
+
+	c.Check(truthy(nil), Equals, false)
+
+	c.Check(truthy("foobar"), Equals, true)
+	c.Check(truthy(-1), Equals, true)
+	c.Check(truthy(gin.H{}), Equals, true)
+}

api/db.go

@@ -0,0 +1,180 @@
+package api
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/db/cleanup
+func apiDbCleanup(c *gin.Context) {
+
+	resources := []string{string(task.AllResourcesKey)}
+	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var err error
+
+		collectionFactory := context.NewCollectionFactory()
+
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
+
+		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
+		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if e != nil {
+				return e
+			}
+
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+			if published.SourceKind != deb.SourceLocalRepo {
+				return nil
+			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
+
+			for _, component := range published.Components() {
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// ... and compare it to the list of all packages
+		out.Printf("Loading list of all packages...")
+		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+		// delete packages that are no longer referenced
+		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
+
+		// database can't err as collection factory already constructed
+		db, _ := context.Database()
+
+		if toDelete.Len() > 0 {
+			batch := db.CreateBatch()
+			toDelete.ForEach(func(ref []byte) error {
+				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				return nil
+			})
+
+			err = batch.Write()
+			if err != nil {
+				return nil, fmt.Errorf("unable to write to DB: %s", err)
+			}
+		}
+
+		// now, build a list of files that should be present in Repository (package pool)
+		out.Printf("Building list of files referenced by packages...")
+		referencedFiles := make([]string, 0, existingPackageRefs.Len())
+
+		err = existingPackageRefs.ForEach(func(key []byte) error {
+			pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+			if err2 != nil {
+				tail := ""
+				return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+			}
+			paths, err2 := pkg.FilepathList(context.PackagePool())
+			if err2 != nil {
+				return err2
+			}
+			referencedFiles = append(referencedFiles, paths...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sort.Strings(referencedFiles)
+
+		// build a list of files in the package pool
+		out.Printf("Building list of files in package pool...")
+		existingFiles, err := context.PackagePool().FilepathList(out)
+		if err != nil {
+			return nil, fmt.Errorf("unable to collect file paths: %s", err)
+		}
+
+		// find files which are in the pool but not referenced by packages
+		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+		// delete files that are no longer referenced
+		out.Printf("Deleting unreferenced files (%d)...", len(filesToDelete))
+
+		countFilesToDelete := len(filesToDelete)
+		taskDetail := struct {
+			TotalNumberOfPackagesToDelete     int
+			RemainingNumberOfPackagesToDelete int
+		}{
+			countFilesToDelete, countFilesToDelete,
+		}
+		detail.Store(taskDetail)
+
+		if countFilesToDelete > 0 {
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return nil, err
+				}
+
+				taskDetail.RemainingNumberOfPackagesToDelete--
+				detail.Store(taskDetail)
+				totalSize += size
+			}
+
+			out.Printf("Disk space freed: %s...", utils.HumanBytes(totalSize))
+		}
+
+		out.Printf("Compacting database...")
+		return nil, db.CompactDB()
+	})
+}

api/files.go

@@ -0,0 +1,186 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gin-gonic/gin"
+)
+
+func verifyPath(path string) bool {
+	path = filepath.Clean(path)
+	for _, part := range strings.Split(path, string(filepath.Separator)) {
+		if part == ".." || part == "." {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+func verifyDir(c *gin.Context) bool {
+	if !verifyPath(c.Params.ByName("dir")) {
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		return false
+	}
+
+	return true
+}
+
+// GET /files
+func apiFilesListDirs(c *gin.Context) {
+	list := []string{}
+
+	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == context.UploadPath() {
+			return nil
+		}
+
+		if info.IsDir() {
+			list = append(list, filepath.Base(path))
+			return filepath.SkipDir
+		}
+
+		return nil
+	})
+
+	if err != nil && !os.IsNotExist(err) {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// POST /files/:dir/
+func apiFilesUpload(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	stored := []string{}
+
+	for _, files := range c.Request.MultipartForm.File {
+		for _, file := range files {
+			src, err := file.Open()
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer src.Close()
+
+			destPath := filepath.Join(path, filepath.Base(file.Filename))
+			dst, err := os.Create(destPath)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+			defer dst.Close()
+
+			_, err = io.Copy(dst, src)
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
+		}
+	}
+
+	c.JSON(200, stored)
+
+}
+
+// GET /files/:dir
+func apiFilesListFiles(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	list := []string{}
+	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path == root {
+			return nil
+		}
+
+		list = append(list, filepath.Base(path))
+
+		return nil
+	})
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.AbortWithError(404, err)
+		} else {
+			c.AbortWithError(500, err)
+		}
+		return
+	}
+
+	c.JSON(200, list)
+}
+
+// DELETE /files/:dir
+func apiFilesDeleteDir(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}
+
+// DELETE /files/:dir/:name
+func apiFilesDeleteFile(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	if !verifyPath(c.Params.ByName("name")) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	if err != nil {
+		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
+			c.AbortWithError(500, err)
+			return
+		}
+	}
+
+	c.JSON(200, gin.H{})
+}

api/gpg.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/gpg
+func apiGPGAddKey(c *gin.Context) {
+	var b struct {
+		Keyserver   string
+		GpgKeyID    string
+		GpgKeyArmor string
+		Keyring     string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	var err error
+	args := []string{"--no-default-keyring"}
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+	args = append(args, "--keyring", keyring)
+	if len(b.Keyserver) > 0 {
+		args = append(args, "--keyserver", b.Keyserver)
+	}
+	if len(b.GpgKeyArmor) > 0 {
+		var tempdir string
+		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+		defer os.RemoveAll(tempdir)
+
+		keypath := filepath.Join(tempdir, "key")
+		keyfile, e := os.Create(keypath)
+		if e != nil {
+			c.AbortWithError(400, e)
+			return
+		}
+		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
+			c.AbortWithError(400, e)
+		}
+		args = append(args, "--import", keypath)
+
+	}
+	if len(b.GpgKeyID) > 0 {
+		keys := strings.Fields(b.GpgKeyID)
+		args = append(args, "--recv-keys")
+		args = append(args, keys...)
+	}
+
+	finder := pgp.GPG1Finder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	// it might happened that we have a situation with an erroneous
+	// gpg command (e.g. when GpgKeyID and GpgKeyArmor is set).
+	// there is no error handling for such as gpg will do this for us
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	cmd.Stdout = os.Stdout
+	if err = cmd.Run(); err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -0,0 +1,74 @@
+package api
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"os/exec"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+func apiGraph(c *gin.Context) {
+	var (
+		err    error
+		output []byte
+	)
+
+	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
+	factory := context.NewCollectionFactory()
+
+	graph, err := deb.BuildGraph(factory, layout)
+	if err != nil {
+		c.JSON(500, err)
+		return
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
+	command := exec.Command("dot", "-T"+ext)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err = command.Output()
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		return
+	}
+
+	mimeType := mime.TypeByExtension("." + ext)
+	if mimeType == "" {
+		mimeType = "application/octet-stream"
+	}
+
+	c.Data(200, mimeType, output)
+}

api/middleware.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+)
+
+// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
+// See https://prometheus.io/docs/practices/naming/#labels
+func getBasePath(c *gin.Context) string {
+	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+}
+
+func getURLSegment(url string, idx int) string {
+	var urlSegments = strings.Split(url, "/")
+
+	// Remove segment at index 0 because it's an empty string
+	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
+	return fmt.Sprintf("/%s", segmentAtIndex)
+}
+
+func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		g.WithLabelValues(c.Request.Method, pathFunc(c)).Inc()
+		defer g.WithLabelValues(c.Request.Method, pathFunc(c)).Dec()
+		c.Next()
+	}
+}
+
+func instrumentHandlerCounter(counter *prometheus.CounterVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		counter.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Inc()
+	}
+}
+
+func instrumentHandlerRequestSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(float64(c.Request.ContentLength))
+	}
+}
+
+func instrumentHandlerResponseSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		var responseSize = math.Max(float64(c.Writer.Size()), 0)
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(responseSize)
+	}
+}
+
+func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		now := time.Now()
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
+	}
+}

api/mirror.go

@@ -0,0 +1,556 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
+	if ignoreSignatures {
+		return nil, nil
+	}
+
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring()
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+// GET /api/mirrors
+func apiMirrorsList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	result := []*deb.RemoteRepo{}
+	collection.ForEach(func(repo *deb.RemoteRepo) error {
+		result = append(result, repo)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors
+func apiMirrorsCreate(c *gin.Context) {
+	var err error
+	var b struct {
+		Name               string `binding:"required"`
+		ArchiveURL         string `binding:"required"`
+		Distribution       string
+		Filter             string
+		Components         []string
+		Architectures      []string
+		Keyrings           []string
+		DownloadSources    bool
+		DownloadUdebs      bool
+		DownloadInstaller  bool
+		FilterWithDeps     bool
+		SkipComponentCheck bool
+		IgnoreSignatures   bool
+	}
+
+	b.DownloadSources = context.Config().DownloadSourcePackages
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+	b.Architectures = context.ArchitecturesList()
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
+		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+	}
+
+	if b.Filter != "" {
+		_, err = query.Parse(b.Filter)
+		if err != nil {
+			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			return
+		}
+	}
+
+	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		return
+	}
+
+	repo.Filter = b.Filter
+	repo.FilterWithDeps = b.FilterWithDeps
+	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.DownloadSources = b.DownloadSources
+	repo.DownloadUdebs = b.DownloadUdebs
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	downloader := context.NewDownloader(nil)
+	err = repo.Fetch(downloader, verifier)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		return
+	}
+
+	err = collection.Add(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// DELETE /api/mirrors/:name
+func apiMirrorsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	mirrorCollection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	repo, err := mirrorCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+			}
+		}
+
+		err = mirrorCollection.Drop(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}
+
+// GET /api/mirrors/:name
+func apiMirrorsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/mirrors/:name/packages
+func apiMirrorsPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	if repo.LastDownloadDate.IsZero() {
+		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		return
+	}
+
+	reflist := repo.RefList()
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}
+
+// PUT /api/mirrors/:name
+func apiMirrorsUpdate(c *gin.Context) {
+	var (
+		err    error
+		remote *deb.RemoteRepo
+	)
+
+	var b struct {
+		Name                 string
+		ArchiveURL           string
+		Filter               string
+		Architectures        []string
+		Components           []string
+		Keyrings             []string
+		FilterWithDeps       bool
+		DownloadSources      bool
+		DownloadUdebs        bool
+		SkipComponentCheck   bool
+		IgnoreChecksums      bool
+		IgnoreSignatures     bool
+		ForceUpdate          bool
+		SkipExistingPackages bool
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	remote, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	b.Name = remote.Name
+	b.DownloadUdebs = remote.DownloadUdebs
+	b.DownloadSources = remote.DownloadSources
+	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.FilterWithDeps = remote.FilterWithDeps
+	b.Filter = remote.Filter
+	b.Architectures = remote.Architectures
+	b.Components = remote.Components
+
+	log.Printf("%s: Starting mirror update\n", b.Name)
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Name != remote.Name {
+		_, err = collection.ByName(b.Name)
+		if err == nil {
+			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			return
+		}
+	}
+
+	if b.DownloadUdebs != remote.DownloadUdebs {
+		if remote.IsFlat() && b.DownloadUdebs {
+			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			return
+		}
+	}
+
+	if b.ArchiveURL != "" {
+		remote.SetArchiveRoot(b.ArchiveURL)
+	}
+
+	remote.Name = b.Name
+	remote.DownloadUdebs = b.DownloadUdebs
+	remote.DownloadSources = b.DownloadSources
+	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.FilterWithDeps = b.FilterWithDeps
+	remote.Filter = b.Filter
+	remote.Architectures = b.Architectures
+	remote.Components = b.Components
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	resources := []string{string(remote.Key())}
+	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+		downloader := context.NewDownloader(out)
+		err := remote.Fetch(downloader, verifier)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if !b.ForceUpdate {
+			err = remote.CheckLock()
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if remote.Filter != "" {
+			var filterQuery deb.PackageQuery
+
+			filterQuery, err = query.Parse(remote.Filter)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			_, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		defer func() {
+			// on any interruption, unlock the mirror
+			e := context.ReOpenDatabase()
+			if e == nil {
+				remote.MarkAsIdle()
+				collection.Update(remote)
+			}
+		}()
+
+		remote.MarkAsUpdating()
+		err = collection.Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		context.GoContextHandleSignals()
+
+		count := len(queue)
+		taskDetail := struct {
+			TotalDownloadSize         int64
+			RemainingDownloadSize     int64
+			TotalNumberOfPackages     int
+			RemainingNumberOfPackages int
+		}{
+			downloadSize, downloadSize, count, count,
+		}
+		detail.Store(taskDetail)
+
+		downloadQueue := make(chan int)
+		taskFinished := make(chan *deb.PackageDownloadTask)
+
+		var (
+			errors  []string
+			errLock sync.Mutex
+		)
+
+		pushError := func(err error) {
+			errLock.Lock()
+			errors = append(errors, err.Error())
+			errLock.Unlock()
+		}
+
+		go func() {
+			for idx := range queue {
+				select {
+				case downloadQueue <- idx:
+				case <-context.Done():
+					return
+				}
+			}
+
+			close(downloadQueue)
+		}()
+
+		// update of task details need to be done in order
+		go func() {
+			for {
+				task, ok := <-taskFinished
+				if !ok {
+					return
+				}
+
+				taskDetail.RemainingDownloadSize -= task.File.Checksums.Size
+				taskDetail.RemainingNumberOfPackages--
+				detail.Store(taskDetail)
+			}
+		}()
+
+		log.Printf("%s: Spawning background processes...\n", b.Name)
+		var wg sync.WaitGroup
+		for i := 0; i < context.Config().DownloadConcurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case idx, ok := <-downloadQueue:
+						if !ok {
+							return
+						}
+
+						task := &queue[idx]
+
+						var e error
+
+						// provision download location
+						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// download file...
+						e = context.Downloader().DownloadWithChecksum(
+							context,
+							remote.PackageURL(task.File.DownloadURL()).String(),
+							task.TempDownPath,
+							&task.File.Checksums,
+							b.IgnoreChecksums)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						task.Done = true
+						taskFinished <- task
+					case <-context.Done():
+						return
+					}
+
+				}
+			}()
+		}
+
+		// Wait for all download goroutines to finish
+		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		wg.Wait()
+		log.Printf("%s: Background processes finished\n", b.Name)
+		close(taskFinished)
+
+		for idx := range queue {
+
+			atask := &queue[idx]
+
+			if !atask.Done {
+				// download not finished yet
+				continue
+			}
+
+			// and import it back to the pool
+			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+			}
+
+			// update "attached" files if any
+			for _, additionalAtask := range atask.Additional {
+				additionalAtask.File.PoolPath = atask.File.PoolPath
+				additionalAtask.File.Checksums = atask.File.Checksums
+			}
+		}
+
+		select {
+		case <-context.Done():
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+
+		if len(errors) > 0 {
+			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+		}
+
+		log.Printf("%s: Finalizing download\n", b.Name)
+		remote.FinalizeDownload(collectionFactory, out)
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}

api/mirror_test.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MirrorSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&MirrorSuite{})
+
+func (s *MirrorSuite) TestGetMirrors(c *C) {
+	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}
+
+func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
+	response, _ := s.HTTPRequest("DELETE", "/api/mirrors/does-not-exist", nil)
+	c.Check(response.Code, Equals, 404)
+	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
+}
+
+func (s *MirrorSuite) TestCreateMirror(c *C) {
+	c.ExpectFailure("Need to mock downloads")
+	body, err := json.Marshal(gin.H{
+		"Name":       "dummy",
+		"ArchiveURL": "foobar",
+	})
+	c.Assert(err, IsNil)
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Equals, "")
+}

api/packages.go

@@ -0,0 +1,17 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/packages/:key
+func apiPackagesShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, p)
+}

api/publish.go

@@ -0,0 +1,395 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// SigningOptions is a shared between publish API GPG options structure
+type SigningOptions struct {
+	Skip           bool
+	Batch          bool
+	GpgKey         string
+	Keyring        string
+	SecretKeyring  string
+	Passphrase     string
+	PassphraseFile string
+}
+
+func getSigner(options *SigningOptions) (pgp.Signer, error) {
+	if options.Skip {
+		return nil, nil
+	}
+
+	signer := context.GetSigner()
+	signer.SetKey(options.GpgKey)
+	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
+	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
+	signer.SetBatch(options.Batch)
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+}
+
+// Replace '_' with '/' and double '__' with single '_'
+func parseEscapedPath(path string) string {
+	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	if result == "" {
+		result = "."
+	}
+	return result
+}
+
+// GET /publish
+func apiPublishList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	result := make([]*deb.PublishedRepo, 0, collection.Len())
+
+	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
+		err := collection.LoadComplete(repo, collectionFactory)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, result)
+}
+
+// POST /publish/:prefix
+func apiPublishRepoOrSnapshot(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+
+	var b struct {
+		SourceKind string `binding:"required"`
+		Sources    []struct {
+			Component string
+			Name      string `binding:"required"`
+		} `binding:"required"`
+		Distribution         string
+		Label                string
+		Origin               string
+		NotAutomatic         string
+		ButAutomaticUpgrades string
+		ForceOverwrite       bool
+		SkipContents         *bool
+		SkipBz2              *bool
+		Architectures        []string
+		Signing              SigningOptions
+		AcquireByHash        *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	if len(b.Sources) == 0 {
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
+		return
+	}
+
+	var components []string
+	var names []string
+	var sources []interface{}
+	var resources []string
+	collectionFactory := context.NewCollectionFactory()
+
+	if b.SourceKind == "snapshot" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := collectionFactory.SnapshotCollection()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+			names = append(names, source.Name)
+
+			snapshot, err = snapshotCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			resources = append(resources, string(snapshot.ResourceKey()))
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			sources = append(sources, snapshot)
+		}
+	} else if b.SourceKind == deb.SourceLocalRepo {
+		var localRepo *deb.LocalRepo
+
+		localCollection := collectionFactory.LocalRepoCollection()
+
+		for _, source := range b.Sources {
+			components = append(components, source.Component)
+			names = append(names, source.Name)
+
+			localRepo, err = localCollection.ByName(source.Name)
+			if err != nil {
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				return
+			}
+
+			resources = append(resources, string(localRepo.Key()))
+			err = localCollection.LoadComplete(localRepo)
+			if err != nil {
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+			}
+
+			sources = append(sources, localRepo)
+		}
+	} else {
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
+		return
+	}
+
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+		return
+	}
+
+	resources = append(resources, string(published.Key()))
+	collection := collectionFactory.PublishedRepoCollection()
+
+	taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskDetail := task.PublishDetail{
+			Detail: detail,
+		}
+		publishOutput := &task.PublishOutput{
+			Progress:      out,
+			PublishDetail: taskDetail,
+		}
+
+		if b.Origin != "" {
+			published.Origin = b.Origin
+		}
+		if b.NotAutomatic != "" {
+			published.NotAutomatic = b.NotAutomatic
+		}
+		if b.ButAutomaticUpgrades != "" {
+			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+		}
+		published.Label = b.Label
+
+		published.SkipContents = context.Config().SkipContentsPublishing
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+
+		published.SkipBz2 = context.Config().SkipBz2Publishing
+		if b.SkipContents != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+
+		duplicate := collection.CheckDuplicate(published)
+		if duplicate != nil {
+			collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
+		}
+
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
+		}
+
+		err = collection.Add(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: published}, nil
+	})
+}
+
+// PUT /publish/:prefix/:distribution
+func apiPublishUpdateSwitch(c *gin.Context) {
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	var b struct {
+		ForceOverwrite bool
+		Signing        SigningOptions
+		SkipContents   *bool
+		SkipBz2        *bool
+		SkipCleanup    *bool
+		Snapshots      []struct {
+			Component string `binding:"required"`
+			Name      string `binding:"required"`
+		}
+		AcquireByHash *bool
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	signer, err := getSigner(&b.Signing)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+	err = collection.LoadComplete(published, collectionFactory)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+		return
+	}
+
+	var updatedComponents []string
+	var updatedSnapshots []string
+	var resources []string
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		if len(b.Snapshots) > 0 {
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			return
+		}
+		updatedComponents = published.Components()
+		for _, component := range updatedComponents {
+			published.UpdateLocalRepo(component)
+		}
+	} else if published.SourceKind == "snapshot" {
+		publishedComponents := published.Components()
+		for _, snapshotInfo := range b.Snapshots {
+			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				return
+			}
+
+			snapshotCollection := collectionFactory.SnapshotCollection()
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				c.AbortWithError(404, err2)
+				return
+			}
+
+			err2 = snapshotCollection.LoadComplete(snapshot)
+			if err2 != nil {
+				c.AbortWithError(500, err2)
+				return
+			}
+
+			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
+			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+			updatedSnapshots = append(updatedSnapshots, snapshot.Name)
+		}
+	} else {
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
+		return
+	}
+
+	if b.SkipContents != nil {
+		published.SkipContents = *b.SkipContents
+	}
+
+	if b.SkipBz2 != nil {
+		published.SkipBz2 = *b.SkipBz2
+	}
+
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
+	resources = append(resources, string(published.Key()))
+	taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = collection.Update(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		if b.SkipCleanup == nil || !*b.SkipCleanup {
+			err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+				context.GetPublishedStorage(storage), collectionFactory, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
+	})
+}
+
+// DELETE /publish/:prefix/:distribution
+func apiPublishDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
+
+	param := parseEscapedPath(c.Params.ByName("prefix"))
+	storage, prefix := deb.ParsePrefix(param)
+	distribution := c.Params.ByName("distribution")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
+
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(published.Key())}
+
+	taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.Remove(context, storage, prefix, distribution,
+			collectionFactory, out, force, skipCleanup)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}

api/repos.go

@@ -0,0 +1,495 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/repos
+func apiReposList(c *gin.Context) {
+	result := []*deb.LocalRepo{}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	collection.ForEach(func(r *deb.LocalRepo) error {
+		result = append(result, r)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/repos
+func apiReposCreate(c *gin.Context) {
+	var b struct {
+		Name                string `binding:"required"`
+		Comment             string
+		DefaultDistribution string
+		DefaultComponent    string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	repo := deb.NewLocalRepo(b.Name, b.Comment)
+	repo.DefaultComponent = b.DefaultComponent
+	repo.DefaultDistribution = b.DefaultDistribution
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	err := collection.Add(repo)
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// PUT /api/repos/:name
+func apiReposEdit(c *gin.Context) {
+	var b struct {
+		Name                *string
+		Comment             *string
+		DefaultDistribution *string
+		DefaultComponent    *string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	if b.Name != nil {
+		_, err := collection.ByName(*b.Name)
+		if err == nil {
+			// already exists
+			c.AbortWithError(404, err)
+			return
+		}
+		repo.Name = *b.Name
+	}
+	if b.Comment != nil {
+		repo.Comment = *b.Comment
+	}
+	if b.DefaultDistribution != nil {
+		repo.DefaultDistribution = *b.DefaultDistribution
+	}
+	if b.DefaultComponent != nil {
+		repo.DefaultComponent = *b.DefaultComponent
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/repos/:name
+func apiReposShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+// DELETE /api/repos/:name
+func apiReposDrop(c *gin.Context) {
+	force := c.Request.URL.Query().Get("force") == "1"
+	name := c.Params.ByName("name")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.ByLocalRepo(repo)
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+	})
+}
+
+// GET /api/repos/:name/packages
+func apiReposPackagesShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, repo.RefList(), collectionFactory)
+}
+
+// Handler for both add and delete
+func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
+	var b struct {
+		PackageRefs []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	repo, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Loading packages...\n")
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			var p *deb.Package
+
+			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
+				}
+
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = cb(list, p, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+			}
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
+}
+
+// POST /repos/:name/packages
+func apiReposPackagesAdd(c *gin.Context) {
+	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Adding package %s\n", p.Name)
+		return list.Add(p)
+	})
+}
+
+// DELETE /repos/:name/packages
+func apiReposPackagesDelete(c *gin.Context) {
+	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Removing package %s\n", p.Name)
+		list.Remove(p)
+		return nil
+	})
+}
+
+// POST /repos/:name/file/:dir/:file
+func apiReposPackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposPackageFromDir(c)
+}
+
+// POST /repos/:name/file/:dir
+func apiReposPackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
+
+	if !verifyDir(c) {
+		return
+	}
+
+	dirParam := c.Params.ByName("dir")
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var taskName string
+	var sources []string
+	if fileParam == "" {
+		taskName = fmt.Sprintf("Add packages from dir %s to repo %s", dirParam, name)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+		taskName = fmt.Sprintf("Add package %s from dir %s to repo %s", fileParam, dirParam, name)
+	}
+
+	resources := []string{string(repo.Key())}
+	resources = append(resources, sources...)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		verifier := context.GetVerifier()
+
+		var (
+			packageFiles, failedFiles    []string
+			otherFiles                   []string
+			processedFiles, failedFiles2 []string
+			reporter                     = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+			list *deb.PackageList
+		)
+
+		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+		failedFiles = append(failedFiles, failedFiles2...)
+		processedFiles = append(processedFiles, otherFiles...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+
+		if !noRemove {
+			processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+			for _, file := range processedFiles {
+				err := os.Remove(file)
+				if err != nil {
+					reporter.Warning("unable to remove file %s: %s", file, err)
+				}
+			}
+
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
+	})
+}
+
+// POST /repos/:name/include/:dir/:file
+func apiReposIncludePackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposIncludePackageFromDir(c)
+}
+
+// POST /repos/:name/include/:dir
+func apiReposIncludePackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
+	acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
+	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
+
+	repoTemplateString := c.Params.ByName("name")
+	collectionFactory := context.NewCollectionFactory()
+
+	if !verifyDir(c) {
+		return
+	}
+
+	var sources []string
+	var taskName string
+	dirParam := c.Params.ByName("dir")
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	if fileParam == "" {
+		taskName = fmt.Sprintf("Include packages from changes files in dir %s to repo matching template %s", dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
+	} else {
+		taskName = fmt.Sprintf("Include packages from changes file %s from dir %s to repo matching template %s", fileParam, dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+	}
+
+	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
+		return
+	}
+
+	var resources []string
+	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+		resources = append(resources, task.AllLocalReposResourcesKey)
+	} else {
+		// repo template string is simple text so only use resource key of specific repository
+		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
+
+		resources = append(resources, string(repo.Key()))
+	}
+	resources = append(resources, sources...)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var (
+			err                       error
+			verifier                  = context.GetVerifier()
+			changesFiles              []string
+			failedFiles, failedFiles2 []string
+			reporter                  = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+		)
+
+		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+		_, failedFiles2, err = deb.ImportChangesFiles(
+			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+		failedFiles = append(failedFiles, failedFiles2...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import changes files: %s", err)
+		}
+
+		if !noRemoveFiles {
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
+
+	})
+}

api/router.go

@@ -0,0 +1,163 @@
+package api
+
+import (
+	"net/http"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+var context *ctx.AptlyContext
+
+func apiMetricsGet() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
+	}
+}
+
+// Router returns prebuilt with routes http.Handler
+func Router(c *ctx.AptlyContext) http.Handler {
+	context = c
+
+	router := gin.Default()
+	router.UseRawPath = true
+	router.Use(gin.ErrorLogger())
+
+	if c.Config().EnableMetricsEndpoint {
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+	}
+
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		dbRequests = make(chan dbRequest)
+
+		go acquireDatabase()
+
+		router.Use(func(c *gin.Context) {
+			var err error
+
+			errCh := make(chan error)
+			dbRequests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
+			if err != nil {
+				c.AbortWithError(500, err)
+				return
+			}
+
+			defer func() {
+				dbRequests <- dbRequest{releasedb, errCh}
+				err = <-errCh
+				if err != nil {
+					c.AbortWithError(500, err)
+				}
+			}()
+
+			c.Next()
+		})
+	}
+
+	root := router.Group("/api")
+
+	{
+		if c.Config().EnableMetricsEndpoint {
+			root.GET("/metrics", apiMetricsGet())
+		}
+		root.GET("/version", apiVersion)
+	}
+
+	{
+		root.GET("/repos", apiReposList)
+		root.POST("/repos", apiReposCreate)
+		root.GET("/repos/:name", apiReposShow)
+		root.PUT("/repos/:name", apiReposEdit)
+		root.DELETE("/repos/:name", apiReposDrop)
+
+		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+
+		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
+		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+	}
+
+	{
+		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+	}
+
+	{
+		root.GET("/mirrors", apiMirrorsList)
+		root.GET("/mirrors/:name", apiMirrorsShow)
+		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		root.POST("/mirrors", apiMirrorsCreate)
+		root.PUT("/mirrors/:name", apiMirrorsUpdate)
+		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+	}
+
+	{
+		root.POST("/gpg/key", apiGPGAddKey)
+	}
+
+	{
+		root.GET("/files", apiFilesListDirs)
+		root.POST("/files/:dir", apiFilesUpload)
+		root.GET("/files/:dir", apiFilesListFiles)
+		root.DELETE("/files/:dir", apiFilesDeleteDir)
+		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		root.GET("/publish", apiPublishList)
+		root.POST("/publish", apiPublishRepoOrSnapshot)
+		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+	}
+
+	{
+		root.GET("/snapshots", apiSnapshotsList)
+		root.POST("/snapshots", apiSnapshotsCreate)
+		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		root.GET("/snapshots/:name", apiSnapshotsShow)
+		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+	}
+
+	{
+		root.GET("/packages/:key", apiPackagesShow)
+	}
+
+	{
+		root.GET("/graph.:ext", apiGraph)
+	}
+	{
+		root.POST("/db/cleanup", apiDbCleanup)
+	}
+	{
+		root.GET("/tasks", apiTasksList)
+		root.POST("/tasks-clear", apiTasksClear)
+		root.GET("/tasks-wait", apiTasksWait)
+		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		root.GET("/tasks/:id/output", apiTasksOutputShow)
+		root.GET("/tasks/:id/detail", apiTasksDetailShow)
+		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		root.GET("/tasks/:id", apiTasksShow)
+		root.DELETE("/tasks/:id", apiTasksDelete)
+		root.POST("/tasks-dummy", apiTasksDummy)
+	}
+
+	return router
+}

api/snapshot.go

@@ -0,0 +1,399 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /api/snapshots
+func apiSnapshotsList(c *gin.Context) {
+	SortMethodString := c.Request.URL.Query().Get("sort")
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	if SortMethodString == "" {
+		SortMethodString = "name"
+	}
+
+	result := []*deb.Snapshot{}
+	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		result = append(result, snapshot)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors/:name/snapshots/
+func apiSnapshotsCreateFromMirror(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.RemoteRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	repo, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
+		}
+
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+// POST /api/snapshots
+func apiSnapshotsCreate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name            string `binding:"required"`
+		Description     string
+		SourceSnapshots []string
+		PackageRefs     []string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Description == "" {
+		if len(b.SourceSnapshots)+len(b.PackageRefs) == 0 {
+			b.Description = "Created as empty"
+		}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	var resources []string
+
+	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+
+	for i := range b.SourceSnapshots {
+		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			c.AbortWithError(500, err)
+			return
+		}
+
+		resources = append(resources, string(sources[i].ResourceKey()))
+	}
+
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		list := deb.NewPackageList()
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+				}
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = list.Add(p)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+			}
+		}
+
+		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+	})
+}
+
+// POST /api/repos/:name/snapshots
+func apiSnapshotsCreateFromRepository(c *gin.Context) {
+	var (
+		err      error
+		repo     *deb.LocalRepo
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string `binding:"required"`
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	repo, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+// PUT /api/snapshots/:name
+func apiSnapshotsUpdate(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	var b struct {
+		Name        string
+		Description string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
+
+	snapshot, err = collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		_, err := collection.ByName(b.Name)
+		if err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+		}
+
+		if b.Name != "" {
+			snapshot.Name = b.Name
+		}
+
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
+
+		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: snapshot}, nil
+	})
+}
+
+// GET /api/snapshots/:name
+func apiSnapshotsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	c.JSON(200, snapshot)
+}
+
+// DELETE /api/snapshots/:name
+func apiSnapshotsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	resources := []string{string(snapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Delete snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.BySnapshot(snapshot)
+
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
+		}
+
+		if !force {
+			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
+			}
+		}
+
+		err = snapshotCollection.Drop(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
+}
+
+// GET /api/snapshots/:name/diff/:withSnapshot
+func apiSnapshotsDiff(c *gin.Context) {
+	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshotA, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotA)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshotB)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	result := []deb.PackageDiff{}
+
+	for _, pdiff := range diff {
+		if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+			continue
+		}
+
+		result = append(result, pdiff)
+	}
+
+	c.JSON(200, result)
+}
+
+// GET /api/snapshots/:name/packages
+func apiSnapshotsSearchPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	snapshot, err := collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	err = collection.LoadComplete(snapshot)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	showPackages(c, snapshot.RefList(), collectionFactory)
+}

api/task.go

@@ -0,0 +1,155 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /tasks
+func apiTasksList(c *gin.Context) {
+	list := context.TaskList()
+	c.JSON(200, list.GetTasks())
+}
+
+// POST /tasks-clear
+func apiTasksClear(c *gin.Context) {
+	list := context.TaskList()
+	list.Clear()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks-wait
+func apiTasksWait(c *gin.Context) {
+	list := context.TaskList()
+	list.Wait()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks/:id/wait
+func apiTasksWaitForTaskByID(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	task, err := list.WaitForTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id
+func apiTasksShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var task task.Task
+	task, err = list.GetTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id/output
+func apiTasksOutputShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var output string
+	output, err = list.GetTaskOutputByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// GET /tasks/:id/detail
+func apiTasksDetailShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var detail interface{}
+	detail, err = list.GetTaskDetailByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, detail)
+}
+
+// GET /tasks/:id/return_value
+func apiTasksReturnValueShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err := list.GetTaskReturnValueByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// DELETE /tasks/:id
+func apiTasksDelete(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var delTask task.Task
+	delTask, err = list.DeleteTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, delTask)
+}
+
+// POST /tasks-dummy
+func apiTasksDummy(c *gin.Context) {
+	resources := []string{"dummy"}
+	taskName := fmt.Sprintf("Dummy task")
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Dummy task started\n")
+		detail.Store([]int{1, 2, 3})
+		out.Printf("Dummy task finished\n")
+		return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
+	})
+}

api/task_test.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aptly-dev/aptly/task"
+
+	. "gopkg.in/check.v1"
+)
+
+type TaskSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&TaskSuite{})
+
+func (s *TaskSuite) TestTasksDummy(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
+	c.Check(response.Code, Equals, 418)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+}
+
+func (s *TaskSuite) TestTasksDummyAsync(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
+	err = json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
+}
+
+func (s *TaskSuite) TestTaskDelete(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
+	// Give the task time to start
+	time.Sleep(time.Second)
+	response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
+	c.Check(response.Code, Equals, 200)
+}
+
+func (s *TaskSuite) TestTasksClear(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	var ts []task.Task
+	err = json.Unmarshal(response.Body.Bytes(), &ts)
+	c.Assert(err, IsNil)
+	c.Check(len(ts), Equals, 1)
+	c.Check(ts[0].State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "null")
+}

aptly-api.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository API
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/api/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
+
+[Install]
+WantedBy=multi-user.target

aptly.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository server
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/commands/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
+
+[Install]
+WantedBy=multi-user.target

aptly/interfaces.go

@@ -0,0 +1,177 @@
+// Package aptly provides common infrastructure that doesn't depend directly on
+// Debian or CentOS
+package aptly
+
+import (
+	"context"
+	"io"
+	"os"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/utils"
+)
+
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
+// PackagePool is asbtraction of package pool storage.
+//
+// PackagePool stores all the package files, deduplicating them.
+type PackagePool interface {
+	// Verify checks whether file exists in the pool and fills back checksum info
+	//
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
+	// FilepathList returns file paths of all the files in the pool
+	FilepathList(progress Progress) ([]string, error)
+	// Remove deletes file in package pool returns its size
+	Remove(path string) (size int64, err error)
+}
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
+}
+
+// PublishedStorage is abstraction of filesystem storing all published repositories
+type PublishedStorage interface {
+	// MkDir creates directory recursively under public path
+	MkDir(path string) error
+	// PutFile puts file into published storage at specified path
+	PutFile(path string, sourceFilename string) error
+	// RemoveDirs removes directory structure under public path
+	RemoveDirs(path string, progress Progress) error
+	// Remove removes single file under public path
+	Remove(path string) error
+	// LinkFromPool links package file from pool to dist's pool location
+	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	// Filelist returns list of files under prefix
+	Filelist(prefix string) ([]string, error)
+	// RenameFile renames (moves) file
+	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
+}
+
+// FileSystemPublishedStorage is published storage on filesystem
+type FileSystemPublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+}
+
+// PublishedStorageProvider is a thing that returns PublishedStorage by name
+type PublishedStorageProvider interface {
+	// GetPublishedStorage returns PublishedStorage by name
+	GetPublishedStorage(name string) PublishedStorage
+}
+
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
+// Progress is a progress displaying entity, it allows progress bars & simple prints
+type Progress interface {
+	// Writer interface to support progress bar ticking
+	io.Writer
+	// Start makes progress start its work
+	Start()
+	// Shutdown shuts down progress display
+	Shutdown()
+	// Flush returns when all queued messages are sent
+	Flush()
+	// InitBar starts progressbar for count bytes or count items
+	InitBar(count int64, isBytes bool, barType BarType)
+	// ShutdownBar stops progress bar and hides it
+	ShutdownBar()
+	// AddBar increments progress for progress bar
+	AddBar(count int)
+	// SetBar sets current position for progress bar
+	SetBar(count int)
+	// Printf does printf but in safe manner: not overwriting progress bar
+	Printf(msg string, a ...interface{})
+	// ColoredPrintf does printf in colored way + newline
+	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
+}
+
+// Downloader is parallel HTTP fetcher
+type Downloader interface {
+	// Download starts new download task
+	Download(ctx context.Context, url string, destination string) error
+	// DownloadWithChecksum starts new download task with checksum verification
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error
+	// GetProgress returns Progress object
+	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
+}
+
+// ChecksumStorageProvider creates ChecksumStorage based on DB
+type ChecksumStorageProvider func(db database.ReaderWriter) ChecksumStorage
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
+}

aptly/report.go

@@ -0,0 +1,67 @@
+package aptly
+
+import (
+	"fmt"
+)
+
+// ResultReporter is abstraction for result reporting from complex processing functions
+type ResultReporter interface {
+	// Warning is non-fatal error message
+	Warning(msg string, a ...interface{})
+	// Removed is signal that something has been removed
+	Removed(msg string, a ...interface{})
+	// Added is signal that something has been added
+	Added(msg string, a ...interface{})
+}
+
+// ConsoleResultReporter is implementation of ResultReporter that prints in colors to console
+type ConsoleResultReporter struct {
+	Progress Progress
+}
+
+// Check interface
+var (
+	_ ResultReporter = &ConsoleResultReporter{}
+)
+
+// Warning is non-fatal error message (yellow)
+func (c *ConsoleResultReporter) Warning(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@y[!]@| @!"+msg+"@|", a...)
+}
+
+// Removed is signal that something has been removed (red)
+func (c *ConsoleResultReporter) Removed(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@r[-]@| "+msg, a...)
+}
+
+// Added is signal that something has been added (green)
+func (c *ConsoleResultReporter) Added(msg string, a ...interface{}) {
+	c.Progress.ColoredPrintf("@g[+]@| "+msg, a...)
+}
+
+// RecordingResultReporter is implementation of ResultReporter that collects all messages
+type RecordingResultReporter struct {
+	Warnings     []string
+	AddedLines   []string `json:"Added"`
+	RemovedLines []string `json:"Removed"`
+}
+
+// Check interface
+var (
+	_ ResultReporter = &RecordingResultReporter{}
+)
+
+// Warning is non-fatal error message
+func (r *RecordingResultReporter) Warning(msg string, a ...interface{}) {
+	r.Warnings = append(r.Warnings, fmt.Sprintf(msg, a...))
+}
+
+// Removed is signal that something has been removed
+func (r *RecordingResultReporter) Removed(msg string, a ...interface{}) {
+	r.RemovedLines = append(r.RemovedLines, fmt.Sprintf(msg, a...))
+}
+
+// Added is signal that something has been added
+func (r *RecordingResultReporter) Added(msg string, a ...interface{}) {
+	r.AddedLines = append(r.AddedLines, fmt.Sprintf(msg, a...))
+}

aptly/version.go

@@ -0,0 +1,7 @@
+package aptly
+
+// Version of aptly (filled in at link time)
+var Version string
+
+// EnableDebug triggers some debugging features
+const EnableDebug = false

azure/azure.go

@@ -0,0 +1,2 @@
+// Package azure handles publishing to Azure Storage
+package azure

azure/azure_test.go

@@ -0,0 +1,12 @@
+package azure
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

azure/public.go

@@ -0,0 +1,375 @@
+package azure
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Azure)
+type PublishedStorage struct {
+	container azblob.ContainerURL
+	prefix    string
+	pathCache map[string]string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+func isEmulatorEndpoint(endpoint string) bool {
+	if h, _, err := net.SplitHostPort(endpoint); err == nil {
+		endpoint = h
+	}
+	if endpoint == "localhost" {
+		return true
+	}
+	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
+	// In this case, eliminate the '[' and ']' in the URL.
+	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
+	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
+		endpoint = endpoint[1 : len(endpoint)-1]
+	}
+	return net.ParseIP(endpoint) != nil
+}
+
+// NewPublishedStorage creates published storage from Azure storage credentials
+func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = "blob.core.windows.net"
+	}
+
+	var url *url.URL
+	if isEmulatorEndpoint(endpoint) {
+		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
+	} else {
+		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
+	result := &PublishedStorage{
+		container: containerURL,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for Azure
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+
+	sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename)
+	if err != nil {
+		return err
+	}
+
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(path, source, sourceMD5)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// putFile uploads file-like object to
+func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
+	path = filepath.Join(storage.prefix, path)
+
+	blob := storage.container.NewBlockBlobURL(path)
+
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
+	}
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
+		}
+	}
+
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob,
+		uploadOptions,
+	)
+
+	return err
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range filelist {
+		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+		if err != nil {
+			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
+		}
+	}
+
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
+	}
+	return err
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
+
+	relPath := filepath.Join(publishedDirectory, fileName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	if storage.pathCache == nil {
+		paths, md5s, err := storage.internalFilelist("")
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
+		}
+
+		storage.pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			storage.pathCache[paths[i]] = md5s[i]
+		}
+	}
+
+	destinationMD5, exists := storage.pathCache[relPath]
+	sourceMD5 := sourceChecksums.MD5
+
+	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+		}
+	}
+
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(relPath, source, sourceMD5)
+	if err == nil {
+		storage.pathCache[relPath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
+	}
+
+	return err
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := storage.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1000,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		}
+
+		marker = listBlob.NextMarker
+
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	paths, _, err := storage.internalFilelist(prefix)
+	return paths, err
+}
+
+// Internal copy or move implementation
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
+	const leaseDuration = 30
+
+	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
+	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
+	}
+	defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
+	srcBlobLeaseID := leaseResp.LeaseID()
+
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	copyStatus := copyResp.CopyStatus()
+	for {
+		if copyStatus == azblob.CopyStatusSuccess {
+			if move {
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
+				return err
+			}
+			return nil
+		} else if copyStatus == azblob.CopyStatusPending {
+			time.Sleep(1 * time.Second)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
+			if err != nil {
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
+			}
+			copyStatus = blobPropsResp.CopyStatus()
+
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
+			if err != nil {
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
+			}
+		} else {
+			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
+		}
+	}
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */)
+}
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		storageError, ok := err.(azblob.StorageError)
+		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+			return false, nil
+		}
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
+	}
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+	}
+	return resp.NewMetadata()["SymLink"], nil
+}

azure/public_test.go

@@ -0,0 +1,371 @@
+package azure
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type PublishedStorageSuite struct {
+	accountName, accountKey, endpoint string
+	storage, prefixedStorage          *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+const testContainerPrefix = "aptlytest-"
+
+func randContainer() string {
+	return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+	if n <= 0 {
+		panic("negative number")
+	}
+	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+	var bytes = make([]byte, n)
+	rand.Read(bytes)
+	for i, b := range bytes {
+		bytes[i] = alphanum[b%byte(len(alphanum))]
+	}
+	return string(bytes)
+}
+
+func (s *PublishedStorageSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+	cnt := s.storage.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	cnt := s.storage.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+	blob := s.storage.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, IsNil)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := ioutil.ReadAll(body)
+	c.Assert(err, IsNil)
+	return data
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+	_, err := s.storage.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, NotNil)
+	storageError, ok := err.(azblob.StorageError)
+	c.Assert(ok, Equals, true)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	hash := md5.Sum(data)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+
+	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b+c.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+	s.AssertNoFile(c, "a/b c.txt")
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestFilelistPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a 1", "a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a b", "a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	err = s.prefixedStorage.Remove("xyz")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlus(c *C) {
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.RenameFile("source.txt", "dest.txt")
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!"))
+
+	exists, err := s.storage.FileExists("source.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
+
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
+
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
+	c.Assert(err, IsNil)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
+	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in azure test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

cmd/api.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdAPI() *commander.Command {
+	return &commander.Command{
+		UsageLine: "api",
+		Short:     "start API server/issue requests",
+		Subcommands: []*commander.Command{
+			makeCmdAPIServe(),
+		},
+	}
+}

cmd/api_serve.go

@@ -0,0 +1,109 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+
+	"github.com/aptly-dev/aptly/api"
+	"github.com/aptly-dev/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyAPIServe(cmd *commander.Command, args []string) error {
+	var (
+		err error
+	)
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+		os.Remove(file)
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdAPIServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyAPIServe,
+		UsageLine: "serve",
+		Short:     "start API HTTP service",
+		Long: `
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
+
+Example:
+
+  $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
+
+	return cmd
+
+}

cmd/cmd.go

@@ -0,0 +1,131 @@
+// Package cmd implements console commands
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
+)
+
+// ListPackagesRefList shows list of packages in PackageRefList
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
+	fmt.Printf("Packages:\n")
+
+	if reflist == nil {
+		return
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	return PrintPackageList(list, "", "  ")
+
+}
+
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
+	if format == "" {
+		return result.ForEachIndexed(func(p *deb.Package) error {
+			context.Progress().Printf(prefix+"%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEachIndexed(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf(prefix+"%s\n", b.String())
+		return nil
+	})
+
+}
+
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
+// RootCommand creates root command in command tree
+func RootCommand() *commander.Command {
+	cmd := &commander.Command{
+		UsageLine: os.Args[0],
+		Short:     "Debian repository management tool",
+		Long: `
+aptly is a tool to create partial and full mirrors of remote
+repositories, manage local repositories, filter them, merge,
+upgrade individual packages, take snapshots and publish them
+back as Debian repositories.
+
+aptly's goal is to establish repeatability and controlled changes
+in a package-centric environment. aptly allows one to fix a set of packages
+in a repository, so that package installation and upgrade becomes
+deterministic. At the same time aptly allows one to perform controlled,
+fine-grained changes in repository contents to transition your
+package environment to new version.`,
+		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
+		Subcommands: []*commander.Command{
+			makeCmdConfig(),
+			makeCmdDb(),
+			makeCmdGraph(),
+			makeCmdMirror(),
+			makeCmdRepo(),
+			makeCmdServe(),
+			makeCmdSnapshot(),
+			makeCmdTask(),
+			makeCmdPublish(),
+			makeCmdVersion(),
+			makeCmdPackage(),
+			makeCmdAPI(),
+		},
+	}
+
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
+	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
+	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
+	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
+	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
+	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
+
+	if aptly.EnableDebug {
+		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")
+		cmd.Flag.String("memprofile", "", "write memory profile to this file")
+		cmd.Flag.String("memstats", "", "write memory stats periodically to this file")
+		cmd.Flag.Duration("meminterval", 100*time.Millisecond, "memory stats dump interval")
+	}
+	return cmd
+}

cmd/config.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdConfig() *commander.Command {
+	return &commander.Command{
+		UsageLine: "config",
+		Short:     "manage aptly configuration",
+		Subcommands: []*commander.Command{
+			makeCmdConfigShow(),
+		},
+	}
+}

cmd/config_show.go

@@ -0,0 +1,39 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/smira/commander"
+)
+
+func aptlyConfigShow(cmd *commander.Command, args []string) error {
+
+	config := context.Config()
+	prettyJSON, err := json.MarshalIndent(config, "", "    ")
+
+	if err != nil {
+		return fmt.Errorf("unable to dump the config file: %s", err)
+	}
+
+	fmt.Println(string(prettyJSON))
+
+	return nil
+}
+
+func makeCmdConfigShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyConfigShow,
+		UsageLine: "show",
+		Short:     "show current aptly's config",
+		Long: `
+Command show displays the current aptly configuration.
+
+Example:
+
+  $ aptly config show
+
+`,
+	}
+	return cmd
+}

cmd/context.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/smira/flag"
+)
+
+var context *ctx.AptlyContext
+
+// ShutdownContext shuts context down
+func ShutdownContext() {
+	context.Shutdown()
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	context.Cleanup()
+}
+
+// InitContext initializes context with default settings
+func InitContext(flags *flag.FlagSet) error {
+	var err error
+
+	if context != nil {
+		panic("context already initialized")
+	}
+
+	context, err = ctx.NewContext(flags)
+
+	return err
+}
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/db.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdDb() *commander.Command {
+	return &commander.Command{
+		UsageLine: "db",
+		Short:     "manage aptly's internal database and package pool",
+		Subcommands: []*commander.Command{
+			makeCmdDbCleanup(),
+			makeCmdDbRecover(),
+		},
+	}
+}

cmd/db_cleanup.go

@@ -0,0 +1,313 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+)
+
+// aptly db cleanup
+func aptlyDbCleanup(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	// collect information about references packages...
+	existingPackageRefs := deb.NewPackageRefList()
+
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
+		}
+
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
+		if published.SourceKind != deb.SourceLocalRepo {
+			return nil
+		}
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+		if e != nil {
+			return e
+		}
+
+		for _, component := range published.Components() {
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	collectionFactory.Flush()
+
+	// ... and compare it to the list of all packages
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
+	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+	// delete packages that are no longer referenced
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
+
+	// database can't err as collection factory already constructed
+	db, _ := context.Database()
+
+	if toDelete.Len() > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			batch := db.CreateBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+			})
+			if err != nil {
+				return fmt.Errorf("unable to delete by key: %s", err)
+			}
+
+			err = batch.Write()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
+	}
+
+	collectionFactory.Flush()
+
+	// now, build a list of files that should be present in Repository (package pool)
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
+	referencedFiles := make([]string, 0, existingPackageRefs.Len())
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
+
+	err = existingPackageRefs.ForEach(func(key []byte) error {
+		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+		if err2 != nil {
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+		}
+		paths, err2 := pkg.FilepathList(context.PackagePool())
+		if err2 != nil {
+			return err2
+		}
+		referencedFiles = append(referencedFiles, paths...)
+		context.Progress().AddBar(1)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(referencedFiles)
+	context.Progress().ShutdownBar()
+
+	// build a list of files in the package pool
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
+	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to collect file paths: %s", err)
+	}
+
+	// find files which are in the pool but not referenced by packages
+	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+	// delete files that are no longer referenced
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
+
+	if len(filesToDelete) > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
+			for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
+			}
+		}
+
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
+	}
+
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
+
+	return err
+}
+
+func makeCmdDbCleanup() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbCleanup,
+		UsageLine: "cleanup",
+		Short:     "cleanup DB and package pool",
+		Long: `
+Database cleanup removes information about unreferenced packages and removes
+files in the package pool that aren't used by packages anymore
+
+Example:
+
+  $ aptly db cleanup
+`,
+	}
+
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
+	return cmd
+}

cmd/db_recover.go

@@ -0,0 +1,40 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+)
+
+// aptly db recover
+func aptlyDbRecover(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	context.Progress().Printf("Recovering database...\n")
+	err = goleveldb.RecoverDB(context.DBPath())
+
+	return err
+}
+
+func makeCmdDbRecover() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbRecover,
+		UsageLine: "recover",
+		Short:     "recover DB after crash",
+		Long: `
+Database recover does its' best to recover the database after a crash.
+It is recommended to backup the DB before running recover.
+
+Example:
+
+  $ aptly db recover
+`,
+	}
+
+	return cmd
+}

cmd/graph.go

@@ -0,0 +1,149 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+)
+
+func aptlyGraph(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	layout := context.Flags().Lookup("layout").Value.String()
+
+	fmt.Printf("Generating graph...\n")
+	collectionFactory := context.NewCollectionFactory()
+	graph, err := deb.BuildGraph(collectionFactory, layout)
+	if err != nil {
+		return err
+	}
+
+	buf := bytes.NewBufferString(graph.String())
+
+	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	if err != nil {
+		return err
+	}
+	tempfile.Close()
+	os.Remove(tempfile.Name())
+
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
+
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	err = command.Start()
+	if err != nil {
+		return fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err)
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		return err
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		return err
+	}
+
+	err = command.Wait()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		_ = os.Remove(tempfilename)
+	}()
+
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		if err = viewer.Start(); err == nil {
+			// Wait for a second so that the visualizer has a chance to
+			// open the input file. This needs to be done even if we're
+			// waiting for the visualizer as it can be just a wrapper that
+			// spawns a browser tab and returns right away.
+			defer func(t <-chan time.Time) {
+				<-t
+			}(time.After(time.Second))
+		}
+	}
+
+	return err
+}
+
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
+func makeCmdGraph() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyGraph,
+		UsageLine: "graph",
+		Short:     "render graph of relationships",
+		Long: `
+Command graph displays relationship between mirrors, local repositories,
+snapshots and published repositories using graphviz package to render
+graph as an image.
+
+Example:
+
+  $ aptly graph
+`,
+	}
+
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
+	return cmd
+}

cmd/mirror.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
+		return nil, nil
+	}
+
+	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring()
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+type keyRingsFlag struct {
+	keyRings []string
+}
+
+func (k *keyRingsFlag) Set(value string) error {
+	k.keyRings = append(k.keyRings, value)
+	return nil
+}
+
+func (k *keyRingsFlag) Get() interface{} {
+	return k.keyRings
+}
+
+func (k *keyRingsFlag) String() string {
+	return strings.Join(k.keyRings, ",")
+}
+
+func makeCmdMirror() *commander.Command {
+	return &commander.Command{
+		UsageLine: "mirror",
+		Short:     "manage mirrors of remote repositories",
+		Subcommands: []*commander.Command{
+			makeCmdMirrorCreate(),
+			makeCmdMirrorList(),
+			makeCmdMirrorShow(),
+			makeCmdMirrorDrop(),
+			makeCmdMirrorUpdate(),
+			makeCmdMirrorRename(),
+			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
+		},
+	}
+}

cmd/mirror_create.go

@@ -0,0 +1,110 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
+	var err error
+	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
+	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+
+	var (
+		mirrorName, archiveURL, distribution string
+		components                           []string
+	)
+
+	mirrorName = args[0]
+	if len(args) == 2 {
+		archiveURL, distribution, components, err = deb.ParsePPA(args[1], context.Config())
+		if err != nil {
+			return err
+		}
+	} else {
+		archiveURL, distribution, components = args[1], args[2], args[3:]
+	}
+
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs, downloadInstaller)
+	if err != nil {
+		return fmt.Errorf("unable to create mirror: %s", err)
+	}
+
+	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
+	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to create mirror: %s", err)
+		}
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	err = repo.Fetch(context.Downloader(), verifier)
+	if err != nil {
+		return fmt.Errorf("unable to fetch mirror: %s", err)
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add mirror: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s successfully added.\nYou can run 'aptly mirror update %s' to download repository contents.\n", repo, repo.Name)
+	return err
+}
+
+func makeCmdMirrorCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorCreate,
+		UsageLine: "create <name> <archive url> <distribution> [<component1> ...]",
+		Short:     "create new mirror",
+		Long: `
+Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
+line format resembles apt utlitily sources.list(5).
+
+PPA urls could specified in short format:
+
+  $ aptly mirror create <name> ppa:<user>/<project>
+
+Example:
+
+  $ aptly mirror create wheezy-main http://mirror.yandex.ru/debian/ wheezy main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-create", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/mirror_drop.go

@@ -0,0 +1,74 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshots := collectionFactory.SnapshotCollection().ByRemoteRepoSource(repo)
+
+		if len(snapshots) > 0 {
+			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
+			for _, snapshot := range snapshots {
+				fmt.Printf(" * %s\n", snapshot)
+			}
+
+			return fmt.Errorf("won't delete mirror with snapshots, use -force to override")
+		}
+	}
+
+	err = collectionFactory.RemoteRepoCollection().Drop(repo)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Mirror `%s` has been removed.\n", repo.Name)
+
+	return err
+}
+
+func makeCmdMirrorDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete mirror",
+		Long: `
+Drop deletes information about remote repository mirror <name>. Package data is not deleted
+(since it could still be used by other mirrors or snapshots).  If mirror is used as source
+to create a snapshot, aptly would refuse to delete such mirror, use flag -force to override.
+
+Example:
+
+  $ aptly mirror drop wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "force mirror deletion even if used by snapshots")
+
+	return cmd
+}

cmd/mirror_edit.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fetchMirror := false
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "filter":
+			repo.Filter = flag.Value.String()
+		case "filter-with-deps":
+			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
+		}
+	})
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
+	if repo.Filter != "" {
+		_, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
+
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Mirror %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdMirrorEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit mirror settings",
+		Long: `
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
+
+Example:
+
+  $ aptly mirror edit -filter=nginx -filter-with-deps some-mirror
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
+	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorList(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorListJSON(cmd, args)
+	}
+
+	return aptlyMirrorListTxt(cmd, args)
+}
+
+func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
+	i := 0
+	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
+			repos[i] = repo.String()
+		}
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
+	} else {
+		if len(repos) > 0 {
+			fmt.Printf("List of mirrors:\n")
+			for _, repo := range repos {
+				fmt.Printf(" * %s\n", repo)
+			}
+
+			fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+		} else {
+			fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+		}
+	}
+	return err
+}
+
+func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
+func makeCmdMirrorList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorList,
+		UsageLine: "list",
+		Short:     "list mirrors",
+		Long: `
+List shows full list of remote repository mirrors.
+
+Example:
+
+  $ aptly mirror list
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display list in JSON format")
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
+	return cmd
+}

cmd/mirror_rename.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyMirrorRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.RemoteRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.RemoteRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdMirrorRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames mirror",
+		Long: `
+Command changes name of the mirror.Mirror name should be unique.
+
+Example:
+
+  $ aptly mirror rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/mirror_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -0,0 +1,158 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorShow(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorShowJSON(cmd, args)
+	}
+
+	return aptlyMirrorShowTxt(cmd, args)
+}
+
+func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
+	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
+	fmt.Printf("Distribution: %s\n", repo.Distribution)
+	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
+	downloadSources := No
+	if repo.DownloadSources {
+		downloadSources = Yes
+	}
+	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := No
+	if repo.DownloadUdebs {
+		downloadUdebs = Yes
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	if repo.Filter != "" {
+		fmt.Printf("Filter: %s\n", repo.Filter)
+		filterWithDeps := No
+		if repo.FilterWithDeps {
+			filterWithDeps = Yes
+		}
+		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
+	}
+	if repo.LastDownloadDate.IsZero() {
+		fmt.Printf("Last update: never\n")
+	} else {
+		fmt.Printf("Last update: %s\n", repo.LastDownloadDate.Format("2006-01-02 15:04:05 MST"))
+		fmt.Printf("Number of packages: %d\n", repo.NumPackages())
+	}
+
+	fmt.Printf("\nInformation from release file:\n")
+	for _, k := range utils.StrMapSortedKeys(repo.Meta) {
+		fmt.Printf("%s: %s\n", k, repo.Meta[k])
+	}
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.LastDownloadDate.IsZero() {
+			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
+		} else {
+			ListPackagesRefList(repo.RefList(), collectionFactory)
+		}
+	}
+
+	return err
+}
+
+func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				repo.Packages = append(repo.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(repo.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdMirrorShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorShow,
+		UsageLine: "show <name>",
+		Short:     "show details about mirror",
+		Long: `
+Shows detailed information about the mirror.
+
+Example:
+
+  $ aptly mirror show wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
+
+	return cmd
+}

cmd/mirror_update.go

@@ -0,0 +1,275 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
+	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	err = repo.Fetch(context.Downloader(), verifier)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	defer func() {
+		// on any interruption, unlock the mirror
+		err = context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			collectionFactory.RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.GoContextHandleSignals()
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
+
+	downloadQueue := make(chan int)
+
+	var (
+		errors  []string
+		errLock sync.Mutex
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
+	}
+
+	go func() {
+		for idx := range queue {
+			select {
+			case downloadQueue <- idx:
+			case <-context.Done():
+				return
+			}
+		}
+		close(downloadQueue)
+	}()
+
+	var wg sync.WaitGroup
+
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case idx, ok := <-downloadQueue:
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						context,
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreMismatch)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					task.Done = true
+				case <-context.Done():
+					return
+				}
+			}
+		}()
+	}
+
+	// Wait for all download goroutines to finish
+	wg.Wait()
+
+	context.Progress().ShutdownBar()
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
+
+	for idx := range queue {
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(collectionFactory, context.Progress())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	return err
+}
+
+func makeCmdMirrorUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorUpdate,
+		UsageLine: "update <name>",
+		Short:     "update mirror",
+		Long: `
+Updates remote mirror (downloads package files and meta information). When mirror is created,
+this command should be run for the first time to fetch mirror contents. This command can be
+run multiple times to get updated repository contents. If interrupted, command can be safely restarted.
+
+Example:
+
+  $ aptly mirror update wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
+	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
+	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,64 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	if len(args) == 1 {
+		q, err = query.Parse(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search [<package-query>]",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query.
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,144 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+
+	return err
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.Flags().Lookup("with-files").Value.Get().(bool)
+	withReferences := context.Flags().Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
+			for _, f := range p.Files() {
+				var path string
+				path, err = f.GetPoolPath(packagePool)
+				if err != nil {
+					return err
+				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p, collectionFactory)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matching query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -0,0 +1,43 @@
+package cmd
+
+import (
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
+		return nil, nil
+	}
+
+	signer := context.GetSigner()
+	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
+	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+
+}
+
+func makeCmdPublish() *commander.Command {
+	return &commander.Command{
+		UsageLine: "publish",
+		Short:     "manage published repositories",
+		Subcommands: []*commander.Command{
+			makeCmdPublishDrop(),
+			makeCmdPublishList(),
+			makeCmdPublishRepo(),
+			makeCmdPublishSnapshot(),
+			makeCmdPublishSwitch(),
+			makeCmdPublishUpdate(),
+			makeCmdPublishShow(),
+		},
+	}
+}

cmd/publish_drop.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		collectionFactory, context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	context.Progress().Printf("\nPublished repository has been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishDrop,
+		UsageLine: "drop <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "remove published repository",
+		Long: `
+Command removes whatever has been published under specified <prefix>,
+publishing <endpoint> and <distribution> name.
+
+Example:
+
+    $ aptly publish drop wheezy
+`,
+	}
+
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
+	return cmd
+}

cmd/publish_list.go

@@ -0,0 +1,134 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishList(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishListJSON(cmd, args)
+	}
+
+	return aptlyPublishListTxt(cmd, args)
+}
+
+func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+
+	collectionFactory := context.NewCollectionFactory()
+	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
+
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		if raw {
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
+		} else {
+			published = append(published, repo.String())
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Strings(published)
+
+	if raw {
+		for _, info := range published {
+			fmt.Printf("%s\n", info)
+		}
+	} else {
+		if len(published) == 0 {
+			fmt.Printf("No snapshots/local repos have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
+			return err
+		}
+
+		fmt.Printf("Published repositories:\n")
+
+		for _, description := range published {
+			fmt.Printf("  * %s\n", description)
+		}
+	}
+
+	return err
+}
+
+func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		repos = append(repos, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GetPath() < repos[j].GetPath()
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
+func makeCmdPublishList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishList,
+		UsageLine: "list",
+		Short:     "list of published repositories",
+		Long: `
+Display list of currently published snapshots.
+
+Example:
+
+    $ aptly publish list
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display list in JSON format")
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
+	return cmd
+}

cmd/publish_repo.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdPublishRepo() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "repo <name> [[<endpoint>:]<prefix>]",
+		Short:     "publish local repository",
+		Long: `
+Command publishes current state of local repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple local
+repositories as the arguments:
+
+    aptly publish repo -component=main,contrib repo-main repo-contrib
+
+It is not recommended to publish local repositories directly unless the
+repository is for testing purposes and changes happen frequently. For
+production usage please take snapshot of repository and publish it
+using publish snapshot command.
+
+Example:
+
+    $ aptly publish repo testing
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-repo", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+
+	return cmd
+}

cmd/publish_show.go

@@ -0,0 +1,126 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishShowJSON(cmd, args)
+	}
+
+	return aptlyPublishShowTxt(cmd, args)
+}
+
+func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for component, sourceID := range repo.Sources {
+		var name string
+		if repo.SourceKind == deb.SourceSnapshot {
+			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == deb.SourceLocalRepo {
+			source, e := collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -0,0 +1,246 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
+	var err error
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	collectionFactory := context.NewCollectionFactory()
+
+	if len(args) < len(components) || len(args) > len(components)+1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	var param string
+	if len(args) == len(components)+1 {
+		param = args[len(components)]
+		args = args[0 : len(args)-1]
+	} else {
+		param = ""
+	}
+	storage, prefix := deb.ParsePrefix(param)
+
+	var (
+		sources = []interface{}{}
+		message string
+	)
+
+	if cmd.Name() == "snapshot" { // nolint: goconst
+		var (
+			snapshot     *deb.Snapshot
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, snapshot)
+			parts = append(parts, snapshot.Name)
+
+			if snapshot.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
+
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Snapshot %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Snapshots %s have", strings.Join(parts, ", "))
+
+		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else if cmd.Name() == "repo" { // nolint: goconst
+		var (
+			localRepo    *deb.LocalRepo
+			emptyWarning = false
+			parts        = []string{}
+		)
+
+		for _, name := range args {
+			localRepo, err = collectionFactory.LocalRepoCollection().ByName(name)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
+			if err != nil {
+				return fmt.Errorf("unable to publish: %s", err)
+			}
+
+			sources = append(sources, localRepo)
+			parts = append(parts, localRepo.Name)
+
+			if localRepo.NumPackages() == 0 {
+				emptyWarning = true
+			}
+		}
+
+		if len(parts) == 1 {
+			message = fmt.Sprintf("Local repo %s has", parts[0])
+		} else {
+			message = fmt.Sprintf("Local repos %s have", strings.Join(parts, ", "))
+
+		}
+
+		if emptyWarning {
+			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
+		}
+	} else {
+		panic("unknown command")
+	}
+
+	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
+	published.Label = context.Flags().Lookup("label").Value.String()
+	published.Suite = context.Flags().Lookup("suite").Value.String()
+
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	published.SkipBz2 = context.Config().SkipBz2Publishing
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
+	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
+	if duplicate != nil {
+		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Add(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	var repoComponents string
+	prefix, repoComponents, distribution = published.Prefix, strings.Join(published.Components(), " "), published.Distribution
+	if prefix == "." {
+		prefix = ""
+	} else if !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	context.Progress().Printf("\n%s been successfully published.\n", message)
+
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
+		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+			localStorage.PublicPath())
+	}
+
+	context.Progress().Printf("Now you can add following line to apt sources:\n")
+	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
+		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
+	}
+	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+	context.Progress().Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
+
+	return err
+}
+
+func makeCmdPublishSnapshot() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshotOrRepo,
+		UsageLine: "snapshot <name> [[<endpoint>:]<prefix>]",
+		Short:     "publish snapshot",
+		Long: `
+Command publishes snapshot as Debian repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Multiple component repository could be published by specifying several
+components split by commas via -component flag and multiple snapshots
+as the arguments:
+
+    aptly publish snapshot -component=main,contrib snap-main snap-contrib
+
+Example:
+
+    $ aptly publish snapshot wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
+	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -0,0 +1,166 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
+	var err error
+
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(args) < len(components)+1 || len(args) > len(components)+2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	var (
+		names    []string
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) == len(components)+2 {
+		param = args[1]
+		names = args[2:]
+	} else {
+		names = args[1:]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != deb.SourceSnapshot {
+		return fmt.Errorf("unable to update: not a snapshot publish")
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	publishedComponents := published.Components()
+	if len(components) == 1 && len(publishedComponents) == 1 && components[0] == "" {
+		components = publishedComponents
+	}
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
+	}
+
+	for i, component := range components {
+		if !utils.StrSliceHasItem(publishedComponents, component) {
+			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+		}
+
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
+
+		published.UpdateSnapshot(component, snapshot)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
+	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishSwitch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSwitch,
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
+		Short:     "update published repository by switching to new snapshot",
+		Long: `
+Command switches in-place published snapshots with new snapshot contents. All
+publishing parameters are preserved (architecture list, distribution,
+component).
+
+For multiple component repositories, flag -component should be given with
+list of components to update. Corresponding snapshots should be given in the
+same order, e.g.:
+
+	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
+
+Example:
+
+    $ aptly publish switch wheezy ppa wheezy-7.5
+
+This command would switch published repository (with one component) named ppa/wheezy
+(prefix ppa, dsitribution wheezy to new snapshot wheezy-7.5).
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
+	return cmd
+}

cmd/publish_update.go

@@ -0,0 +1,124 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+	storage, prefix := deb.ParsePrefix(param)
+
+	var published *deb.PublishedRepo
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	if published.SourceKind != deb.SourceLocalRepo {
+		return fmt.Errorf("unable to update: not a local repository publish")
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	components := published.Components()
+	for _, component := range components {
+		published.UpdateLocalRepo(component)
+	}
+
+	signer, err := getSigner(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
+	if forceOverwrite {
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+			"the same package pool.\n")
+	}
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
+	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+
+	return err
+}
+
+func makeCmdPublishUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishUpdate,
+		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "update published local repository",
+		Long: `
+Command re-publishes (updates) published local repository. <distribution>
+and <prefix> should be occupied with local repository published
+using command aptly publish repo. Update happens in-place with
+minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories
+are updated.
+
+Example:
+
+    $ aptly publish update wheezy ppa
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
+	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
+	return cmd
+}

cmd/repo.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdRepo() *commander.Command {
+	return &commander.Command{
+		UsageLine: "repo",
+		Short:     "manage local package repositories",
+		Subcommands: []*commander.Command{
+			makeCmdRepoAdd(),
+			makeCmdRepoCopy(),
+			makeCmdRepoCreate(),
+			makeCmdRepoDrop(),
+			makeCmdRepoEdit(),
+			makeCmdRepoImport(),
+			makeCmdRepoList(),
+			makeCmdRepoMove(),
+			makeCmdRepoRemove(),
+			makeCmdRepoShow(),
+			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
+		},
+	}
+}

cmd/repo_add.go

@@ -0,0 +1,114 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoAdd(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+
+	verifier := context.GetVerifier()
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	context.Progress().Printf("Loading packages...\n")
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+
+	var packageFiles, otherFiles, failedFiles []string
+
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
+
+	var processedFiles, failedFiles2 []string
+
+	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+		collectionFactory.PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		collectionFactory.ChecksumCollection)
+	failedFiles = append(failedFiles, failedFiles2...)
+	if err != nil {
+		return fmt.Errorf("unable to import package files: %s", err)
+	}
+
+	processedFiles = append(processedFiles, otherFiles...)
+
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+	err = collectionFactory.LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to save: %s", err)
+	}
+
+	if context.Flags().Lookup("remove-files").Value.Get().(bool) {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err = os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoAdd,
+		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		Short:     "add packages to local repository",
+		Long: `
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
+patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
+to the database. Files would be imported to internal package pool. For source packages, all required files are
+added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
+
+Example:
+
+  $ aptly repo add testing myapp-0.1.2.deb incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+
+	return cmd
+}

cmd/repo_copy.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoCopy() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "copy <src-name> <dst-name> <package-query> ...",
+		Short:     "copy packages between local repositories",
+		Long: `
+Command copy copies packages matching <package-query> from local repo
+<src-name> to local repo <dst-name>.
+
+Example:
+
+  $ aptly repo copy testing stable 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-copy", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't copy, just show what would be copied")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_create.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoCreate(cmd *commander.Command, args []string) error {
+	var err error
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	repo := deb.NewLocalRepo(args[0], context.Flags().Lookup("comment").Value.String())
+	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
+	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
+
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	err = collectionFactory.LocalRepoCollection().Add(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add local repo: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s successfully added.\nYou can run 'aptly repo add %s ...' to add packages to repository.\n", repo, repo.Name)
+	return err
+}
+
+func makeCmdRepoCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoCreate,
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
+		Short:     "create local repository",
+		Long: `
+Create local package repository. Repository would be empty when
+created, packages could be added from files, copied or moved from
+another local repository or imported from the mirror.
+
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
+Example:
+
+  $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
+
+	return cmd
+}

cmd/repo_drop.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published := collectionFactory.PublishedRepoCollection().ByLocalRepo(repo)
+	if len(published) > 0 {
+		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
+		for _, repo := range published {
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: local repo is published")
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshots := collectionFactory.SnapshotCollection().ByLocalRepoSource(repo)
+
+		if len(snapshots) > 0 {
+			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
+			for _, snapshot := range snapshots {
+				fmt.Printf(" * %s\n", snapshot)
+			}
+
+			return fmt.Errorf("won't delete local repo with snapshots, use -force to override")
+		}
+	}
+
+	err = collectionFactory.LocalRepoCollection().Drop(repo)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Local repo `%s` has been removed.\n", repo.Name)
+
+	return err
+}
+
+func makeCmdRepoDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete local repository",
+		Long: `
+Drop information about deletions from local repo. Package data is not deleted
+(since it could be still used by other mirrors or snapshots).
+
+Example:
+
+  $ aptly repo drop local-repo
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "force local repo deletion even if used by snapshots")
+
+	return cmd
+}

cmd/repo_edit.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/AlekSi/pointer"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoEdit(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	var uploadersFile *string
+
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
+
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
+	}
+
+	err = collectionFactory.LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
+	fmt.Printf("Local repo %s successfully updated.\n", repo)
+	return err
+}
+
+func makeCmdRepoEdit() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoEdit,
+		UsageLine: "edit <name>",
+		Short:     "edit properties of local repository",
+		Long: `
+Command edit allows one to change metadata of local repository:
+comment, default distribution and component.
+
+Example:
+
+  $ aptly repo edit -distribution=wheezy testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-edit", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+	cmd.Flag.String("distribution", "", "default distribution when publishing")
+	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
+
+	return cmd
+}

cmd/repo_import.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoImport() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "import <src-mirror> <dst-repo> <package-query> ...",
+		Short:     "import packages from mirror to local repository",
+		Long: `
+Command import looks up packages matching <package-query> in mirror <src-mirror>
+and copies them to local repo <dst-repo>.
+
+Example:
+
+  $ aptly repo import wheezy-main testing nginx
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-import", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't import, just show what would be imported")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_include.go

@@ -0,0 +1,112 @@
+package cmd
+
+import (
+	"fmt"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = context.GetVerifier()
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+	collectionFactory := context.NewCollectionFactory()
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, failedFiles2 []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
+		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+		context.PackagePool(), collectionFactory.ChecksumCollection,
+		uploaders, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include <file.changes>|<directory> ...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -0,0 +1,122 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoList(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoListJSON(cmd, args)
+	}
+
+	return aptlyRepoListTxt(cmd, args)
+}
+
+func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+
+	collectionFactory := context.NewCollectionFactory()
+	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
+	i := 0
+	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if raw {
+			repos[i] = repo.Name
+		} else {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
+		}
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Strings(repos)
+
+	if raw {
+		for _, repo := range repos {
+			fmt.Printf("%s\n", repo)
+		}
+	} else {
+		if len(repos) > 0 {
+			fmt.Printf("List of local repos:\n")
+			for _, repo := range repos {
+				fmt.Println(repo)
+			}
+
+			fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+		} else {
+			fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+		}
+	}
+
+	return err
+}
+
+func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
+func makeCmdRepoList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoList,
+		UsageLine: "list",
+		Short:     "list local repositories",
+		Long: `
+List command shows full list of local package repositories.
+
+Example:
+
+  $ aptly repo list
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display list in JSON format")
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+
+	return cmd
+}

cmd/repo_move.go

@@ -0,0 +1,193 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	command := cmd.Name()
+
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	var (
+		srcRefList *deb.PackageRefList
+		srcRepo    *deb.LocalRepo
+	)
+
+	if command == "copy" || command == "move" { // nolint: goconst
+		srcRepo, err = collectionFactory.LocalRepoCollection().ByName(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		if srcRepo.UUID == dstRepo.UUID {
+			return fmt.Errorf("unable to %s: source and destination are the same", command)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		srcRefList = srcRepo.RefList()
+	} else if command == "import" { // nolint: goconst
+		var srcRemoteRepo *deb.RemoteRepo
+
+		srcRemoteRepo, err = collectionFactory.RemoteRepoCollection().ByName(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		if srcRemoteRepo.RefList() == nil {
+			return fmt.Errorf("unable to %s: mirror not updated", command)
+		}
+
+		srcRefList = srcRemoteRepo.RefList()
+	} else {
+		panic("unexpected command")
+	}
+
+	context.Progress().Printf("Loading packages...\n")
+
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	srcList.PrepareIndex()
+
+	var architecturesList []string
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+
+	if withDeps {
+		dstList.PrepareIndex()
+
+		// Calculate architectures
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = dstList.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i := 0; i < len(args)-2; i++ {
+		queries[i], err = query.Parse(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+	}
+
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	var verb string
+
+	if command == "move" { // nolint: goconst
+		verb = "moved"
+	} else if command == "copy" { // nolint: goconst
+		verb = "copied"
+	} else if command == "import" { // nolint: goconst
+		verb = "imported"
+	}
+
+	err = toProcess.ForEach(func(p *deb.Package) error {
+		err = dstList.Add(p)
+		if err != nil {
+			return err
+		}
+
+		if command == "move" { // nolint: goconst
+			srcList.Remove(p)
+		}
+		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
+	} else {
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
+
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		if command == "move" { // nolint: goconst
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
+
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
+			if err != nil {
+				return fmt.Errorf("unable to save: %s", err)
+			}
+		}
+	}
+
+	return err
+}
+
+func makeCmdRepoMove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "move <src-name> <dst-name> <package-query> ...",
+		Short:     "move packages between local repositories",
+		Long: `
+Command move moves packages matching <package-query> from local repo
+<src-name> to local repo <dst-name>.
+
+Example:
+
+  $ aptly repo move testing stable 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-move", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't move, just show what would be moved")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_remove.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoRemove(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	context.Progress().Printf("Loading packages...\n")
+
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	queries := make([]deb.PackageQuery, len(args)-1)
+	for i := 0; i < len(args)-1; i++ {
+		queries[i], err = query.Parse(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to remove: %s", err)
+		}
+	}
+
+	list.PrepareIndex()
+	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	toRemove.ForEach(func(p *deb.Package) error {
+		list.Remove(p)
+		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+		return nil
+	})
+
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
+	} else {
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+	}
+
+	return err
+}
+
+func makeCmdRepoRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRemove,
+		UsageLine: "remove <name> <package-query> ...",
+		Short:     "remove packages from local repository",
+		Long: `
+Commands removes packages matching <package-query> from local repository
+<name>. If removed packages are not referenced by other repos or
+snapshots, they can be removed completely (including files) by running
+'aptly db cleanup'.
+
+Example:
+
+  $ aptly repo remove testing 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't remove, just show what would be removed")
+
+	return cmd
+}

cmd/repo_rename.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyRepoRename(cmd *commander.Command, args []string) error {
+	var (
+		err  error
+		repo *deb.LocalRepo
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.LocalRepoCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.LocalRepoCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
+	}
+
+	repo.Name = newName
+	err = collectionFactory.LocalRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdRepoRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames local repository",
+		Long: `
+Command changes name of the local repo. Local repo name should be unique.
+
+Example:
+
+  $ aptly repo rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/repo_search.go

@@ -0,0 +1,29 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/repo_show.go

@@ -0,0 +1,121 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoShow(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoShowJSON(cmd, args)
+	}
+
+	return aptlyRepoShowTxt(cmd, args)
+}
+
+func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", repo.Name)
+	fmt.Printf("Comment: %s\n", repo.Comment)
+	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
+	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
+	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		ListPackagesRefList(repo.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	packageList := []string{}
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err == nil {
+				packageList = list.FullNames()
+			}
+		}
+
+		sort.Strings(packageList)
+	}
+
+	// merge the repo object with the package list
+	var output []byte
+	if output, err = json.MarshalIndent(struct {
+		*deb.LocalRepo
+		Packages []string
+	}{repo, packageList}, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdRepoShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoShow,
+		UsageLine: "show <name>",
+		Short:     "show details about local repository",
+		Long: `
+Show command shows full information about local package repository.
+
+ex:
+  $ aptly repo show testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.Bool("with-packages", false, "show list of packages")
+
+	return cmd
+}

cmd/run.go

@@ -0,0 +1,47 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*ctx.FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			ctx.Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+
+	return
+}

cmd/serve.go

@@ -0,0 +1,131 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyServe(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().RootDir)
+	if err != nil {
+		return err
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if collectionFactory.PublishedRepoCollection().Len() == 0 {
+		fmt.Printf("No published repositories, unable to serve.\n")
+		return nil
+	}
+
+	listen := context.Flags().Lookup("listen").Value.String()
+
+	listenHost, listenPort, err := net.SplitHostPort(listen)
+
+	if err != nil {
+		return fmt.Errorf("wrong -listen specification: %s", err)
+	}
+
+	if listenHost == "" {
+		listenHost, err = os.Hostname()
+		if err != nil {
+			listenHost = "localhost"
+		}
+	}
+
+	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
+
+	sources := make(sort.StringSlice, 0, collectionFactory.PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, collectionFactory.PublishedRepoCollection().Len())
+
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		if e != nil {
+			return e
+		}
+
+		sources = append(sources, repo.String())
+		published[repo.String()] = repo
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	sort.Strings(sources)
+
+	for _, source := range sources {
+		repo := published[source]
+
+		prefix := repo.Prefix
+		if prefix == "." {
+			prefix = ""
+		} else {
+			prefix += "/"
+		}
+
+		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
+			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
+
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
+			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
+				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
+		}
+	}
+
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
+	ShutdownContext()
+
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	err = http.ListenAndServe(listen, http.FileServer(http.Dir(publicPath)))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+	return nil
+}
+
+func makeCmdServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyServe,
+		UsageLine: "serve",
+		Short:     "HTTP serve published repositories",
+		Long: `
+Command serve starts embedded HTTP server (not suitable for real production usage) to serve
+contents of public/ subdirectory of aptly's root that contains published repositories.
+
+Example:
+
+  $ aptly serve -listen=:8080
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+
+	return cmd
+}

cmd/snapshot.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdSnapshot() *commander.Command {
+	return &commander.Command{
+		UsageLine: "snapshot",
+		Short:     "manage snapshots of repositories",
+		Subcommands: []*commander.Command{
+			makeCmdSnapshotCreate(),
+			makeCmdSnapshotList(),
+			makeCmdSnapshotShow(),
+			makeCmdSnapshotVerify(),
+			makeCmdSnapshotPull(),
+			makeCmdSnapshotDiff(),
+			makeCmdSnapshotMerge(),
+			makeCmdSnapshotDrop(),
+			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
+		},
+	}
+}

cmd/snapshot_create.go

@@ -0,0 +1,110 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
+		// aptly snapshot create snap from mirror mirror
+		var repo *deb.RemoteRepo
+
+		repoName, snapshotName := args[3], args[0]
+
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(repoName)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = deb.NewSnapshotFromRepository(snapshotName, repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
+		// aptly snapshot create snap from repo repo
+		var repo *deb.LocalRepo
+
+		localRepoName, snapshotName := args[3], args[0]
+
+		repo, err = collectionFactory.LocalRepoCollection().ByName(localRepoName)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = deb.NewSnapshotFromLocalRepo(snapshotName, repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+	} else if len(args) == 2 && args[1] == "empty" {
+		// aptly snapshot create snap empty
+		snapshotName := args[0]
+
+		packageList := deb.NewPackageList()
+
+		snapshot = deb.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
+	} else {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to add snapshot: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", snapshot.Name, snapshot.Name)
+
+	return err
+}
+
+func makeCmdSnapshotCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotCreate,
+		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		Short:     "creates snapshot of mirror (local repository) contents",
+		Long: `
+Command create <name> from mirror makes persistent immutable snapshot of remote
+repository mirror. Snapshot could be published or further modified using
+merge, pull and other aptly features.
+
+Command create <name> from repo makes persistent immutable snapshot of local
+repository. Snapshot could be processed as mirror snapshots, and mixed with
+snapshots of remote mirrors.
+
+Command create <name> empty creates empty snapshot that could be used as a
+basis for snapshot pull operations, for example. As snapshots are immutable,
+creating one empty snapshot should be enough.
+
+Example:
+
+  $ aptly snapshot create wheezy-main-today from mirror wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_diff.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name-a> snapshot
+	snapshotA, err := collectionFactory.SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot A: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot A: %s", err)
+	}
+
+	// Load <name-b> snapshot
+	snapshotB, err := collectionFactory.SnapshotCollection().ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot B: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot B: %s", err)
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		return fmt.Errorf("unable to calculate diff: %s", err)
+	}
+
+	if len(diff) == 0 {
+		context.Progress().Printf("Snapshots are identical.\n")
+	} else {
+		context.Progress().Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
+		for _, pdiff := range diff {
+			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+				continue
+			}
+
+			var verA, verB, pkg, arch, code string
+
+			if pdiff.Left == nil {
+				verA = "-"
+				verB = pdiff.Right.Version
+				pkg = pdiff.Right.Name
+				arch = pdiff.Right.Architecture
+			} else {
+				pkg = pdiff.Left.Name
+				arch = pdiff.Left.Architecture
+				verA = pdiff.Left.Version
+				if pdiff.Right == nil {
+					verB = "-"
+				} else {
+					verB = pdiff.Right.Version
+				}
+			}
+
+			if pdiff.Left == nil {
+				code = "@g+@|"
+			} else {
+				if pdiff.Right == nil {
+					code = "@r-@|"
+				} else {
+					code = "@y!@|"
+				}
+			}
+
+			context.Progress().ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
+		}
+	}
+
+	return err
+}
+
+func makeCmdSnapshotDiff() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotDiff,
+		UsageLine: "diff <name-a> <name-b>",
+		Short:     "difference between two snapshots",
+		Long: `
+Displays difference in packages between two snapshots. Snapshot is a list
+of packages, so difference between snapshots is a difference between package
+lists. Package could be either completely missing in one snapshot, or package
+is present in both snapshots with different versions.
+
+Example:
+
+    $ aptly snapshot diff -only-matching wheezy-main wheezy-backports
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-diff", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("only-matching", false, "display diff only for matching packages (don't display missing packages)")
+
+	return cmd
+}

cmd/snapshot_drop.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published := collectionFactory.PublishedRepoCollection().BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
+		for _, repo := range published {
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: snapshot is published")
+	}
+
+	force := context.Flags().Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshots := collectionFactory.SnapshotCollection().BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
+			for _, snap := range snapshots {
+				fmt.Printf(" * %s\n", snap)
+			}
+
+			return fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use -force to override")
+		}
+	}
+
+	err = collectionFactory.SnapshotCollection().Drop(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Snapshot `%s` has been dropped.\n", snapshot.Name)
+
+	return err
+}
+
+func makeCmdSnapshotDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete snapshot",
+		Long: `
+Drop removes information about a snapshot. If snapshot is published,
+it can't be dropped.
+
+Example:
+
+    $ aptly snapshot drop wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "remove snapshot even if it was used as source for other snapshots")
+
+	return cmd
+}

cmd/snapshot_filter.go

@@ -0,0 +1,109 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <source> snapshot
+	source, err := collectionFactory.SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = collectionFactory.SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotList(cmd *commander.Command, args []string) error {
+	if len(args) != 0 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlySnapshotListJSON(cmd, args)
+	}
+
+	return aptlySnapshotListTxt(cmd, args)
+}
+
+func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+
+	if raw {
+		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+			fmt.Printf("%s\n", snapshot.Name)
+			return nil
+		})
+	} else {
+		if collection.Len() > 0 {
+			fmt.Printf("List of snapshots:\n")
+
+			err = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+				fmt.Printf(" * %s\n", snapshot.String())
+				return nil
+			})
+
+			if err != nil {
+				return err
+			}
+
+			fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
+		} else {
+			fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+		}
+	}
+
+	return err
+}
+
+func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	collection := context.NewCollectionFactory().SnapshotCollection()
+
+	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
+	i := 0
+	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		jsonSnapshots[i] = snapshot
+		i++
+		return nil
+	})
+	if output, e := json.MarshalIndent(jsonSnapshots, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
+func makeCmdSnapshotList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotList,
+		UsageLine: "list",
+		Short:     "list snapshots",
+		Long: `
+Command list shows full list of snapshots created.
+
+Example:
+
+  $ aptly snapshot list
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display list in JSON format")
+	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
+	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
+
+	return cmd
+}

cmd/snapshot_merge.go

@@ -0,0 +1,92 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	sources := make([]*deb.Snapshot, len(args)-1)
+
+	for i := 0; i < len(args)-1; i++ {
+		sources[i], err = collectionFactory.SnapshotCollection().ByName(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to load snapshot: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
+		if err != nil {
+			return fmt.Errorf("unable to load snapshot: %s", err)
+		}
+	}
+
+	latest := context.Flags().Lookup("latest").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
+
+	if noRemove && latest {
+		return fmt.Errorf("-no-remove and -latest can't be specified together")
+	}
+
+	overrideMatching := !latest && !noRemove
+
+	result := sources[0].RefList()
+	for i := 1; i < len(sources); i++ {
+		result = result.Merge(sources[i].RefList(), overrideMatching, false)
+	}
+
+	if latest {
+		result.FilterLatestRefs()
+	}
+
+	sourceDescription := make([]string, len(sources))
+	for i, s := range sources {
+		sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
+		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
+
+	err = collectionFactory.SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotMerge() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMerge,
+		UsageLine: "merge <destination> <source> [<source>...]",
+		Short:     "merges snapshots",
+		Long: `
+Merge command merges several <source> snapshots into one <destination> snapshot.
+Merge happens from left to right. By default, packages with the same
+name-architecture pair are replaced during merge (package from latest snapshot
+on the list wins).  If run with only one source snapshot, merge copies <source> into
+<destination>.
+
+Example:
+
+    $ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
+`,
+	}
+
+	cmd.Flag.Bool("latest", false, "use only the latest version of each package")
+	cmd.Flag.Bool("no-remove", false, "don't remove duplicate arch/name packages")
+
+	return cmd
+}

cmd/snapshot_pull.go

@@ -0,0 +1,176 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotPull(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 4 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
+	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
+	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name> snapshot
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	// Load <source> snapshot
+	source, err := collectionFactory.SnapshotCollection().ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	context.Progress().Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
+		snapshot, source, args[2])
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+	sourcePackageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+	var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+	for _, arch := range architecturesList {
+		archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-3)
+	for i, arg := range args[3:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+		// Add architecture filter
+		queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+	}
+
+	// Filter with dependencies as requested
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+	result.PrepareIndex()
+
+	alreadySeen := map[string]bool{}
+
+	result.ForEachIndexed(func(pkg *deb.Package) error {
+		key := pkg.Architecture + "_" + pkg.Name
+		_, seen := alreadySeen[key]
+
+		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+		if !noRemove && !seen {
+			// Remove all packages with the same name and architecture
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			for _, p := range pS {
+				packageList.Remove(p)
+				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
+			}
+		}
+
+		// If !allMatches, add only first matching name-arch package
+		if !seen || allMatches {
+			packageList.Add(pkg)
+			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
+		}
+
+		alreadySeen[key] = true
+
+		return nil
+	})
+	alreadySeen = nil
+
+	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
+		context.Progress().Printf("\nNot creating snapshot, as dry run was requested.\n")
+	} else {
+		// Create <destination> snapshot
+		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destination)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		context.Progress().Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+	}
+	return err
+}
+
+func makeCmdSnapshotPull() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotPull,
+		UsageLine: "pull <name> <source> <destination> <package-query> ...",
+		Short:     "pull packages from another snapshot",
+		Long: `
+Command pull pulls new packages along with its' dependencies to snapshot <name>
+from snapshot <source>. Pull can upgrade package version in <name> with
+versions from <source> following dependencies. New snapshot <destination>
+is created as a result of this process. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-pull", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
+	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
+	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+	cmd.Flag.Bool("all-matches", false, "pull all the packages that satisfy the dependency version requirements")
+
+	return cmd
+}

cmd/snapshot_rename.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotRename(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+	)
+
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	oldName, newName := args[0], args[1]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err = collectionFactory.SnapshotCollection().ByName(oldName)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	_, err = collectionFactory.SnapshotCollection().ByName(newName)
+	if err == nil {
+		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
+	}
+
+	snapshot.Name = newName
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s -> %s has been successfully renamed.\n", oldName, newName)
+
+	return err
+}
+
+func makeCmdSnapshotRename() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotRename,
+		UsageLine: "rename <old-name> <new-name>",
+		Short:     "renames snapshot",
+		Long: `
+Command changes name of the snapshot. Snapshot name should be unique.
+
+Example:
+
+  $ aptly snapshot rename wheezy-min wheezy-main
+`,
+	}
+
+	return cmd
+
+}

cmd/snapshot_search.go

@@ -0,0 +1,143 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+	collectionFactory := context.NewCollectionFactory()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" { // nolint: goconst
+		var snapshot *deb.Snapshot
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		var repo *deb.RemoteRepo
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" { // nolint: goconst
+		var repo *deb.LocalRepo
+		repo, err = collectionFactory.LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	if len(args) == 2 {
+		q, err = query.Parse(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
+	}
+
+	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	if result.Len() == 0 {
+		return fmt.Errorf("no results")
+	}
+
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> [<package-query>]",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+If query is not specified, all the packages are displayed.
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -0,0 +1,178 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlySnapshotShow(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlySnapshotShowJSON(cmd, args)
+	}
+
+	return aptlySnapshotShowTxt(cmd, args)
+}
+
+func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", snapshot.Name)
+	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
+	fmt.Printf("Description: %s\n", snapshot.Description)
+	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+	if len(snapshot.SourceIDs) > 0 {
+		fmt.Printf("Sources:\n")
+		for _, sourceID := range snapshot.SourceIDs {
+			var name string
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = collectionFactory.SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = collectionFactory.RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				name = source.Name
+			}
+
+			if name != "" {
+				fmt.Printf("  %s [%s]\n", name, snapshot.SourceKind)
+			}
+		}
+	}
+
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	snapshot, err := context.NewCollectionFactory().SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include the sources
+	if len(snapshot.SourceIDs) > 0 {
+		for _, sourceID := range snapshot.SourceIDs {
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = context.NewCollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.Snapshots = append(snapshot.Snapshots, source)
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.LocalRepos = append(snapshot.LocalRepos, source)
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = context.NewCollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.RemoteRepos = append(snapshot.RemoteRepos, source)
+			}
+		}
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if snapshot.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(snapshot.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(snapshot, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdSnapshotShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotShow,
+		UsageLine: "show <name>",
+		Short:     "shows details about snapshot",
+		Long: `
+Command show displays full information about a snapshot.
+
+Example:
+
+    $ aptly snapshot show wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.Bool("with-packages", false, "show list of packages")
+
+	return cmd
+}

cmd/snapshot_verify.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	snapshots := make([]*deb.Snapshot, len(args))
+	collectionFactory := context.NewCollectionFactory()
+	for i := range snapshots {
+		snapshots[i], err = collectionFactory.SnapshotCollection().ByName(args[i])
+		if err != nil {
+			return fmt.Errorf("unable to verify: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i])
+		if err != nil {
+			return fmt.Errorf("unable to verify: %s", err)
+		}
+	}
+
+	context.Progress().Printf("Loading packages...\n")
+
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), collectionFactory.PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	sourcePackageList := deb.NewPackageList()
+	err = sourcePackageList.Append(packageList)
+	if err != nil {
+		return fmt.Errorf("unable to merge sources: %s", err)
+	}
+
+	var pL *deb.PackageList
+	for i := 1; i < len(snapshots); i++ {
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		err = sourcePackageList.Append(pL)
+		if err != nil {
+			return fmt.Errorf("unable to merge sources: %s", err)
+		}
+	}
+
+	sourcePackageList.PrepareIndex()
+
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(true)
+	}
+
+	if len(architecturesList) == 0 {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	context.Progress().Printf("Verifying...\n")
+
+	missing, err := packageList.VerifyDependencies(context.DependencyOptions(), architecturesList, sourcePackageList, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to verify dependencies: %s", err)
+	}
+
+	if len(missing) == 0 {
+		context.Progress().Printf("All dependencies are satisfied.\n")
+	} else {
+		context.Progress().Printf("Missing dependencies (%d):\n", len(missing))
+		deps := make([]string, len(missing))
+		i := 0
+		for _, dep := range missing {
+			deps[i] = dep.String()
+			i++
+		}
+
+		sort.Strings(deps)
+
+		for _, dep := range deps {
+			context.Progress().Printf("  %s\n", dep)
+		}
+	}
+
+	return err
+}
+
+func makeCmdSnapshotVerify() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotVerify,
+		UsageLine: "verify <name> [<source> ...]",
+		Short:     "verify dependencies in snapshot",
+		Long: `
+Verify does dependency resolution in snapshot <name>, possibly using additional
+snapshots <source> as dependency sources. All unsatisfied dependencies are
+printed.
+
+Example:
+
+    $ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
+`,
+	}
+
+	return cmd
+}

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,154 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		var finfo os.FileInfo
+		if finfo, err = os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s", filename)
+		}
+
+		fmt.Print("Reading file...\n\n")
+
+		var file *os.File
+		file, err = os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			err = context.ReOpenDatabase()
+			if err != nil {
+				return fmt.Errorf("failed to reopen DB: %s", err)
+			}
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps organise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+	  $ aptly task run
+	  > repo create local
+	  > repo add local pkg1
+	  > publish repo local
+	  > serve
+	  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}