ci: add release notes

and update Releasing.md

.dockerignore

@@ -0,0 +1,10 @@
+.go/
+.git/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+unit.out
+aptly.test
+build/
+dpkgs/

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 240
+ignore = E126,E241,E741,W504
+include =
+    system
+exclude =
+    system/env

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,14 @@
+<!--- Provide a general summary of the issue in the Title above -->
+
+## Detailed Description
+<!--- Provide a detailed description of the change or addition you are proposing -->
+
+## Context
+<!--- Why is this change important to you? How would you use it? -->
+<!--- How can it benefit other users? -->
+
+## Possible Implementation
+<!--- Not obligatory, but suggest an idea for implementing addition or change -->
+
+## Your Environment
+<!--- Include as many relevant details about the environment you experienced the bug in -->

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,22 @@
+Fixes #
+
+## Requirements
+
+All new code should be covered with tests, documentation should be updated. CI should pass.
+
+## Description of the Change
+
+<!--
+
+Why this change is important?
+
+-->
+
+## Checklist
+
+- [ ] unit-test added (if change is algorithm)
+- [ ] functional test added/updated (if change is functional)
+- [ ] man page updated (if applicable)
+- [ ] bash completion updated (if applicable)
+- [ ] documentation updated
+- [ ] author name in `AUTHORS`

.github/workflows/ci.yml

@@ -0,0 +1,304 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  test:
+    name: "Test (Ubuntu 22.04)"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: "Install Packages"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
+
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Run flake8"
+        run: |
+          make flake8
+
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Install Azurite"
+        id: azuright
+        uses: potatoqualitee/azuright@v1.1
+        with:
+          directory: ${{ runner.temp }}
+
+      - name: "Run Unit Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make test
+
+      - name: "Run Benchmark"
+        run: |
+          COVERAGE_DIR=${{ runner.temp }} make bench
+
+      - name: "Run System Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make system-test
+
+      - name: "Merge Code Coverage"
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+
+      - name: "Upload Code Coverage"
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.txt
+
+  ci-debian-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        arch: ["amd64", "i386" , "arm64" , "armhf"]
+        include:
+          - name: "Debian 13/testing"
+            suite: trixie
+            image: debian:trixie-slim
+          - name: "Debian 12/bookworm"
+            suite: bookworm
+            image: debian:bookworm-slim
+          - name: "Debian 11/bullseye"
+            suite: bullseye
+            image: debian:bullseye-slim
+          - name: "Debian 10/buster"
+            suite: buster
+            image: debian:buster-slim
+          - name: "Ubuntu 24.04"
+            suite: noble
+            image: ubuntu:24.04
+          - name: "Ubuntu 22.04"
+            suite: jammy
+            image: ubuntu:22.04
+          - name: "Ubuntu 20.04"
+            suite: focal
+            image: ubuntu:20.04
+    container:
+      image: ${{ matrix.image }}
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
+    steps:
+      - name: "Install packages"
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
+                          binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf \
+                          libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
+                          gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Build Debian packages"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make dpkg DEBARCH=${{ matrix.arch }}
+
+      - name: "Check aptly credentials"
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          found=no
+          if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
+            found=yes
+          fi
+          echo "Aptly credentials available: $found"
+          echo "FOUND=$found" >> $GITHUB_OUTPUT
+        id: aptlycreds
+
+      - name: "Publish CI release to aptly"
+        if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
+
+      - name: "Publish release to aptly"
+        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
+
+  ci-binary-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, freebsd, darwin]
+        goarch: ["386", "amd64", "arm", "arm64"]
+        exclude:
+          - goos: darwin
+            goarch: 386
+          - goos: darwin
+            goarch: arm
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read Go Version"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
+        env:
+          GOBIN: /usr/local/bin
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
+
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
+        if: startsWith(github.event.ref, 'refs/tags')
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+
+  gh-release:
+    name: "Github Release"
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: ci-binary-build
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Download Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: out/
+
+      - name: "Create Release Notes"
+        run: |
+          echo -e "## Changes\n\n" > out/release-notes.md
+          dpkg-parsechangelog -S Changes | tail -n +4 >> out/release-notes.md
+
+      - name: "Release"
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "Aptly Release ${{ steps.releaseversion.outputs.VERSION }}"
+          files: "out/**/aptly_*.zip"
+          body_path: "out/release-notes.md"

.github/workflows/golangci-lint.yml

@@ -0,0 +1,72 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+
+      - name: Install and initialize swagger
+        run: |
+          go install github.com/swaggo/swag/cmd/swag@latest
+          swag init -q --markdownFiles docs
+        shell: sh
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.54.1
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/scripts/upload-artifacts.sh

@@ -0,0 +1,170 @@
+#!/bin/sh
+
+set -e
+
+builds=build/
+packages=${builds}*.deb
+folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
+aptly_user="$APTLY_USER"
+aptly_password="$APTLY_PASSWORD"
+aptly_api="https://aptly-ops.aptly.info"
+version=`make version`
+
+action=$1
+dist=$2
+
+usage() {
+    echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
+    echo "       $0 release" >&2
+}
+
+# repos and publish must be created beforehand:
+#!/bin/sh
+#for dist in buster bullseye bookworm focal jammy noble
+#do
+#        for build in ci release
+#        do
+#            echo
+#            echo "# Creating and publishing $build/$dist"
+#            aptly repo create -distribution=$dist -component=main aptly-$build-$dist
+#            aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
+#                               -distribution=$dist -batch -keyring=aptly.pub \
+#                               aptly-$build-$dist \
+#                               s3:repo.aptly.info:$build
+#       done
+#done
+
+if [ -z "$action" ]; then
+    usage
+    exit 1
+fi
+
+if [ "action" = "ci" ] && [ -z "$dist" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
+    usage
+    echo Error: please set APTLY_USER and APTLY_PASSWORD
+    exit 1
+fi
+
+echo "Publishing version '$version' to $action for $dist...\n"
+
+upload()
+{
+    echo "\nUploading files:"
+    for file in $packages; do
+        echo " - $file"
+        jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+    done
+}
+
+cleanup() {
+    echo "\nCleanup..."
+    jsonret=`curl -fsS -X DELETE  -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+}
+trap cleanup EXIT
+
+sleeptime=5
+retries=60
+wait_task()
+{
+    _id=$1
+    _success=0
+    sleep $sleeptime
+    for t in `seq $retries`
+    do
+        jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
+        _state=`echo $jsonret | jq .State`
+        if [ "$_state" = "2" ]; then
+            _success=1
+            curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id
+            break
+        fi
+        if [ "$_state" = "3" ]; then
+            echo Error: task failed
+            return 1
+        fi
+        sleep $sleeptime
+    done
+    if [ "$_success" -ne 1 ]; then
+        echo Error: task timeout
+        return 1
+    fi
+    return 0
+}
+
+add_packages() {
+    _aptly_repository=$1
+    _folder=$2
+    jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$_aptly_repository/file/$_folder?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: adding packages to $_aptly_repository failed"
+        exit 1
+    fi
+}
+
+update_publish() {
+    _publish=$1
+    _dist=$2
+    jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+        '{"AcquireByHash": true, "MultiDist": true,
+          "Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
+        -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: publish failed"
+        exit 1
+    fi
+}
+
+if [ "$action" = "ci" ]; then
+    if echo "$version" | grep -vq "+"; then
+       # skip ci when on release tag
+       exit 0
+    fi
+
+    aptly_repository=aptly-ci-$dist
+    aptly_published=s3:repo.aptly.info:ci
+
+elif [ "$action" = "release" ]; then
+    aptly_repository=aptly-release-$dist
+    aptly_published=s3:repo.aptly.info:release
+fi
+
+upload
+
+echo "\nAdding packages to $aptly_repository ..."
+add_packages $aptly_repository $folder
+
+echo "\nUpdating published repo $aptly_published ..."
+update_publish $aptly_published $dist
+
+# if [ "$action" = "OBSOLETErelease" ]; then
+#     aptly_repository=aptly-release
+#     aptly_snapshot=aptly-$version
+#     aptly_published=s3:repo.aptly.info:./squeeze
+#
+#     echo "\nAdding packages to $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
+#     echo
+#
+#     echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
+#         "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
+#     echo
+#
+#     echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
+#     curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+#         "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
+#                                    \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
+#                                                  \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
+#         -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
+#     echo
+# fi
+

.gitignore

@@ -2,11 +2,14 @@
 *.o
 *.a
 *.so
+unit.out
 
 # Folders
 _obj
 _test
 
+tmp/
+
 # Architecture specific extensions/prefixes
 *.[568vq]
 [568vq].out
@@ -22,13 +25,52 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
-coverage*.out
+coverage.txt
 
 *.pyc
 
-_vendor/
+xc-out/
+root/
 
-gen
 man/aptly.1.html
-man/aptly.1.ronn
+man/aptly.1.ronn
+
+system/env/
+
+# created by make build for release artifacts
+VERSION
+aptly.test
+
+build/
+
+system/files/aptly2.gpg~
+system/files/aptly2_passphrase.gpg~
+
+*.creds
+
+.go/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+
+# debian
+debian/.debhelper/
+debian/aptly.substvars
+debian/aptly/
+debian/debhelper-build-stamp
+debian/files
+debian/aptly-api/
+debian/*.debhelper
+debian/*.debhelper.log
+debian/aptly-api.substvars
+debian/aptly-dbg.substvars
+debian/aptly-dbg/
+usr/bin/aptly
+dpkgs/
+debian/changelog.dpkg-bak
+
+docs/docs.go
+docs/swagger.json
+docs/swagger.yaml
+docs/swagger.conf

.golangci.yml

@@ -0,0 +1,16 @@
+run:
+  tests: false
+
+
+linters:
+  disable-all: true
+  enable:
+   - goconst
+   - gofmt
+   - goimports
+   - govet
+   - ineffassign
+   - misspell
+   - revive
+   - staticcheck
+   - vetshadow

.travis.yml

@@ -1,35 +0,0 @@
-language: go
-
-go:
-  - 1.3.3
-  - tip
-
-env:
-  global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
-    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
-before_install:
-  - sudo apt-get update -qq
-  - sudo apt-get install -y python-virtualenv graphviz
-  - virtualenv env
-  - . env/bin/activate
-  - pip install boto requests python-swiftclient
-install:
-  - make prepare
-
-
-script: make travis
-
-matrix:
-    allow_failures:
-        - go: tip
-
-
-notifications:
-  webhooks:
-    urls:
-      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false

AUTHORS

@@ -15,3 +15,56 @@ List of contributors, in chronological order:
 * Michael Koval (https://github.com/mkoval)
 * Alexander Guy (https://github.com/alexanderguy)
 * Sebastien Badia (https://github.com/sbadia)
+* Szymon Sobik (https://github.com/sobczyk)
+* Paul Krohn (https://github.com/paul-krohn)
+* Vincent Bernat (https://github.com/vincentbernat)
+* x539 (https://github.com/x539)
+* Phil Frost (https://github.com/bitglue)
+* Benoit Foucher (https://github.com/bentoi)
+* Geoffrey Thomas (https://github.com/geofft)
+* Oliver Sauder (https://github.com/sliverc)
+* Harald Sitter (https://github.com/apachelogger)
+* Johannes Layher (https://github.com/jola5)
+* Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)
+* Shengjing Zhu (https://github.com/zhsj)
+* Nabil Bendafi (https://github.com/nabilbendafi)
+* Raphael Medaer (https://github.com/rmedaer)
+* Raul Benencia (https://github.com/rul)
+* Don Kuntz (https://github.com/dkuntz2)
+* Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)
+* Markus Muellner (https://github.com/mmianl)
+* Chuan Liu (https://github.com/chuan)
+* Samuel Mutel (https://github.com/smutel)
+* Russell Greene (https://github.com/russelltg)
+* Wade Simmons (https://github.com/wadey)
+* Steven Stone (https://github.com/smstone)
+* Josh Bayfield (https://github.com/jbayfield)
+* Boxjan (https://github.com/boxjan)
+* Mauro Regli (https://github.com/reglim)
+* Alexander Zubarev (https://github.com/strike)
+* Nicolas Dostert (https://github.com/acdn-ndostert)
+* Ryan Gonzalez (https://github.com/refi64)
+* Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramon N.Rodriguez (https://github.com/runitonmetal)
+* Golf Hu (https://github.com/hudeng-go)
+* Cookie Fei (https://github.com/wuhuang26)
+* Andrey Loukhnov (https://github.com/aol-nnov)
+* Christoph Fiehe (https://github.com/cfiehe)
+* Blake Kostner (https://github.com/btkostner)
+* Leigh London (https://github.com/leighlondon)
+* Gordian Schoenherr (https://github.com/schoenherrg)

CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

CONTRIBUTING.md

@@ -0,0 +1,253 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
+
+### List of Repositories
+
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly.info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+### Get the Source
+
+To clone the git repo, run the following commands:
+```
+git clone git@github.com:aptly-dev/aptly.git
+cd aptly
+```
+
+## Development Setup
+
+Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
+
+### Docker Development Setup
+
+This section describes the docker setup to start contributing to aptly.
+
+#### Dependencies
+
+Install the following on your development machine:
+- docker
+- make
+- git
+
+##### Docker installation on macOS
+1. Install [Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install/) (or via [Homebrew](https://brew.sh/))
+2. Allow directory sharing
+   - Open Docker Desktop
+   - Go to `Settings → Resources → File Sharing → Virtual File Shares`
+   - Add the aptly git repository path to the shared list (eg. /home/Users/john/aptly)
+
+#### Create docker container
+
+To build the development docker image, run:
+```
+make docker-image
+```
+
+#### Build aptly
+
+To build the aptly in the development docker container, run:
+```
+make docker-build
+```
+
+#### Running aptly commands
+
+To run aptly commands in the development docker container, run:
+```
+make docker-shell
+```
+
+Example:
+```
+$ make docker-shell
+aptly@b43e8473ef81:/work/src$ aptly version
+aptly version: 1.5.0+189+g0fc90dff
+```
+
+#### Running unit tests
+
+In order to run aptly unit tests, enter the following:
+```
+make docker-unit-tests
+```
+
+#### Running system tests
+
+In order to run aptly system tests, enter the following:
+```
+make docker-system-tests
+```
+
+#### Running golangci-lint
+
+In order to run aptly unit tests, run:
+```
+make docker-lint
+```
+
+#### More info
+
+Run `make help` for more information.
+
+
+### Local Development Setup
+
+This section describes local setup to start contributing to aptly.
+
+#### Dependencies
+
+Building aptly requires go version 1.22.
+
+On Debian bookworm with backports enabled, go can be installed with:
+
+    apt install -t bookworm-backports golang-go
+
+#### Building
+
+To build aptly, run:
+
+    make build
+
+Run aptly:
+
+    build/aptly
+
+To install aptly into `$GOPATH/bin`, run:
+
+    make install
+
+#### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+#### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash and Zsh Completion
+
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.

Gomfile

@@ -1,33 +0,0 @@
-gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
-gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
-gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
-gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
-gom 'github.com/AlekSi/pointer', :commit => '5f6d527dae3d678b46fbb20331ddf44e2b841943'
-gom 'github.com/cheggaaa/pb', :commit => '2c1b74620cc58a81ac152ee2d322e28c806d81ed'
-gom 'github.com/gin-gonic/gin', :commit => 'b1758d3bfa09e61ddbc1c9a627e936eec6a170de'
-gom 'github.com/jlaffaye/ftp', :commit => 'fec71e62e457557fbe85cefc847a048d57815d76'
-gom 'github.com/julienschmidt/httprouter', :commit => '46807412fe50aaceb73bb57061c2230fd26a1640'
-gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
-gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
-gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
-gom 'github.com/ncw/swift', :commit => '384ef27c70645e285f8bb9d02276bf654d06027e'
-gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
-gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
-gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
-gom 'github.com/syndtr/gosnappy/snappy', :commit => 'ce8acff4829e0c2458a67ead32390ac0a381c862'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '97e257099d2ab9578151ba85e2641e2cd14d3ca8'
-gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
-gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
-gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
-gom 'golang.org/x/crypto/ssh/terminal', :commit => 'a7ead6ddf06233883deca151dffaef2effbf498f'
-
-group :test do
-    gom 'gopkg.in/check.v1'
-end
-
-group :development do
-   gom 'github.com/golang/lint/golint'
-   gom 'github.com/mattn/goveralls'
-   gom 'github.com/axw/gocov/gocov'
-   gom 'golang.org/x/tools/cmd/cover'
-end

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013-2014 Andrey Smirnov. All rights reserved.
+Copyright 2013-2015 aptly authors. All rights reserved.
 
 MIT License
 

Makefile

@@ -1,86 +1,219 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
-PACKAGES=context database deb files http query swift s3 utils
-ALL_PACKAGES=api aptly context cmd console database deb files http query swift s3 utils
-BINPATH=$(abspath ./_vendor/bin)
-GOM_ENVIRONMENT=-test
-PYTHON?=python
+GOPATH=$(shell go env GOPATH)
+VERSION=$(shell make -s version)
+PYTHON?=python3
+BINPATH?=$(GOPATH)/bin
+GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-ifeq ($(GOVERSION), devel)
-TRAVIS_TARGET=coveralls
-GOM_ENVIRONMENT+=-development
-else
-TRAVIS_TARGET=test
-endif
+# Uncomment to update system test gold files
+# CAPTURE := "--capture"
 
-ifeq ($(TRAVIS), true)
-GOM=$(HOME)/gopath/bin/gom
-else
-GOM=gom
-endif
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-all: test check system-test
+prepare:  ## Install go module dependencies
+	# Prepare go modules
+	go mod verify
+	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-prepare:
-	go get -u github.com/mattn/gom
-	$(GOM) $(GOM_ENVIRONMENT) install
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
+	@reltype=ci ; \
+	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
+	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
+		gittag=`git describe --tags --exact-match 2>/dev/null` ;\
+		if echo "$$gittag" | grep -q '^v[0-9]'; then \
+			reltype=release ; \
+		fi ; \
+	fi ; \
+	echo $$reltype
 
-coverage.out:
-	rm -f coverage.*.out
-	for i in $(PACKAGES); do $(GOM) test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-	rm -f coverage.*.out
+version:  ## Print aptly version
+	@ci="" ; \
+	if [ "`make -s releasetype`" = "ci" ]; then \
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
+	else \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
+	fi
 
-coverage: coverage.out
-	$(GOM) exec go tool cover -html=coverage.out
-	rm -f coverage.out
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
+	# Generate swagger.conf
+	cp docs/swagger.conf.tpl docs/swagger.conf
+	echo "// @version $(VERSION)" >> docs/swagger.conf
 
-check:
-	$(GOM) exec go tool vet -all=true $(ALL_PACKAGES:%=./%)
-	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
+azurite-start:
+	azurite -l /tmp/aptly-azurite & \
+	echo $$! > ~/.azurite.pid
+
+azurite-stop:
+	@kill `cat ~/.azurite.pid`
+
+swagger: swagger-install
+	# Generate swagger docs
+	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+
+etcd-install:
+	# Install etcd
+	test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint: prepare
+	# Install golangci-lint
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
 
 install:
-	$(GOM) build -o $(BINPATH)/aptly
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	# go generate
+	@go generate
+	# go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system-test: install
+test: prepare swagger etcd-install  ## Run unit tests
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
+	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
+	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/aptly-etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
+
+system-test: prepare swagger etcd-install  ## Run system tests
+	# build coverage binary
+	go test -v -coverpkg="./..." -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
+	# Run system tests
+	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
-travis: $(TRAVIS_TARGET) system-test
+bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
-test:
-	$(GOM) test -v ./... -gocheck.v=true
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
-coveralls: coverage.out
-	$(GOM) exec $(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
+	fi
+	# clean
+	rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
+	# Run dpkg-buildpackage
+	@buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
+	echo "$$cmd" ; \
+	$$cmd
+	lintian ../*_$(DEBARCH).changes || true
+	# cleanup
+	@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-test:  ## Run unit tests in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		test \
+		azurite-stop
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		system-test TEST=$(TEST) \
+		azurite-stop
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-package:
-	rm -rf root/
-	mkdir -p root/usr/bin/ root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp $(BINPATH)/aptly root/usr/bin
-	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
-	gzip root/usr/share/man/man1/aptly.1
-	fpm -s dir -t deb -n aptly -v $(VERSION) --url=http://www.aptly.info/ --license=MIT --vendor="Andrey Smirnov <me@smira.ru>" \
-	   -f -m "Andrey Smirnov <me@smira.ru>" --description="Debian repository management tool" --deb-recommends bzip2 --deb-recommends graphviz -C root/ .
-	mv aptly_$(VERSION)_*.deb ~
+man:  ## Create man pages
+	make -C man
 
-src-package:
-	rm -rf aptly-$(VERSION)
-	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
-	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && gom -production install
-	cd aptly-$(VERSION)/src/github.com/smira/aptly && find . \( -name .git -o -name .bzr -o -name .hg \) -print | xargs rm -rf
-	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
-	mkdir -p aptly-$(VERSION)/bash_completion.d
-	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
-	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
-	rm -rf aptly-$(VERSION)
-	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
+clean:  ## remove local build and module cache
+	# Clean all generated and build files
+	test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
+	rm -rf .go/
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: coverage.out
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -1,28 +1,27 @@
-=====
-aptly
-=====
+.. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/aptly-dev/aptly/actions
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
-    :target: https://travis-ci.org/smira/aptly
-
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
-.. image:: http://goreportcard.com/badge/gojp/goreportcard
-    :target: http://goreportcard.com/report/gojp/goreportcard
+.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
+    :target: https://goreportcard.com/report/aptly-dev/aptly
+
+aptly
+=====
 
 Aptly is a swiss army knife for Debian repository management.
 
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
-mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
+open `issues <https://github.com/aptly-dev/aptly/issues>`_ or `discussions <https://github.com/aptly-dev/aptly/discussions>`_.
 
-Aptly features: ("+" means planned features)
+Aptly features:
 
 * make mirrors of remote Debian/Ubuntu repositories, limiting by components/architectures
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
@@ -32,50 +31,107 @@ Aptly features: ("+" means planned features)
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
 * REST API for remote access
-* mirror repositories "as-is" (without resigning with user's key) (+)
-* support for yum repositories (+)
 
-Current limitations:
+Any contributions are welcome! Please see `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
-* translations are not supported yet
+Installation
+=============
 
-Download
---------
+Aptly can be installed on several operating systems.
 
-To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+Debian / Ubuntu
+----------------
 
-    deb http://repo.aptly.info/ squeeze main
+Aptly is provided in the following debian packages:
 
-And import key that is used to sign the release::
+* **aptly**: Includes the main Aptly binary, man pages, and shell completions
+* **aptly-api**: A systemd service for the REST API, using the global /etc/aptly.conf
+* **aptly-dbg**: Debug symbols for troubleshooting
 
-    $ apt-key adv --keyserver keys.gnupg.net --recv-keys E083A3782A194991
+The packages can be installed on official `Debian <https://packages.debian.org/search?keywords=aptly>`_ and `Ubuntu <https://packages.ubuntu.com/search?keywords=aptly>`_ distributions.
 
-After that you can install aptly as any other software package::
+Upstream Debian Packages
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    $ apt-get update
-    $ apt-get install aptly
+If a newer version (not available in Debian/Ubuntu) of aptly is required, upstream debian packages (built from git tags) can be installed as follows:
 
-Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
-Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+Install the following APT key (as root)::
 
-If you would like to use nightly builds (unstable), please use following repository::
+    wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
 
-    deb http://repo.aptly.info/ nightly main
+Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
 
-Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
 
-If you have Go environment set up, you can build aptly from source by running (go 1.3+ required)::
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
-    go get -u github.com/mattn/gom
-    mkdir -p $GOPATH/src/github.com/smira/aptly
-    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
-    cd $GOPATH/src/github.com/smira/aptly
-    gom -production install
-    gom build -o $GOPATH/bin/aptly
+Install aptly packages::
 
-Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
-should work as well, but might fail or produce different result (if external libraries got updated).
+    apt-get update
+    apt-get install aptly
+    apt-get install aptly-api  # REST API systemd service
 
-If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+CI Builds
+~~~~~~~~~~
 
+For testing new features or bugfixes, recent builds are available as CI builds (built from master, may be unstable!) and can be installed as follows:
 
+Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
+
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
+
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
+
+Note: same gpg key is used as for the Upstream Debian Packages.
+
+Other Operating Systems
+------------------------
+
+Binary executables (depends almost only on libc) are available on `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_ for:
+
+- macOS / darwin (amd64, arm64)
+- FreeBSD (amd64, arm64, 386, arm)
+- Generic Linux (amd64, arm64, 386, arm)
+
+Integrations
+=============
+
+Vagrant:
+
+-   `Vagrant configuration <https://github.com/sepulworld/aptly-vagrant>`_ by
+    Zane Williamson, allowing to bring two virtual servers, one with aptly installed
+    and another one set up to install packages from repository published by aptly
+
+Docker:
+
+-    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
+
+With configuration management systems:
+
+-   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
+    (Heavy Water Operations, LLC)
+-   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+    Government Digital Services
+-   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
+    TubeMogul
+-   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
+    Forrest Alvarez and Brian Jackson
+-   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+
+CLI for aptly API:
+
+-   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
+-   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
+
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
+Scala sbt:
+
+-   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar
+
+Molior:
+
+-   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth

Releasing.md

@@ -0,0 +1,16 @@
+# Creating a Release
+
+- create branch release/1.x.y
+- update debian/changelog
+- create PR, merge when approved
+- on updated master, create release:
+```
+version=$(dpkg-parsechangelog -S Version)
+echo Releasing prod version $version
+git tag -a v$version -m 'aptly: release $version'
+git push origin v$version master
+```
+- run swagger locally (`make docker-serve`)
+- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
+- releae www.aptly.info
+- create release announcement on https://github.com/aptly-dev/aptly/discussions

_man/gen.go

@@ -2,16 +2,17 @@ package main
 
 import (
 	"fmt"
-	"github.com/smira/aptly/cmd"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
-	"runtime"
 	"strings"
 	"text/template"
+
+	"github.com/aptly-dev/aptly/cmd"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func allFlags(flags *flag.FlagSet) []*flag.Flag {
@@ -43,22 +44,42 @@ func capitalize(s string) string {
 	return strings.Join(parts, " ")
 }
 
+var authorsS string
+
+func authors() string {
+	return authorsS
+}
+
 func main() {
 	command := cmd.RootCommand()
 	command.UsageLine = "aptly"
 	command.Dispatch(nil)
 
-	_, _File, _, _ := runtime.Caller(0)
-	_File, _ = filepath.Abs(_File)
+	_File, _ := filepath.Abs("./man")
 
 	templ := template.New("man").Funcs(template.FuncMap{
 		"allFlags":    allFlags,
 		"findCommand": findCommand,
 		"toUpper":     strings.ToUpper,
 		"capitalize":  capitalize,
+		"authors":     authors,
 	})
 	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
 
+	authorsF, err := os.Open(filepath.Join(filepath.Dir(_File), "..", "AUTHORS"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsB, err := ioutil.ReadAll(authorsF)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	authorsF.Close()
+
+	authorsS = string(authorsB)
+
 	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
 	if err != nil {
 		log.Fatal(err)

api/api.go

@@ -3,12 +3,18 @@ package api
 
 import (
 	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"net/http"
 	"sort"
-	"time"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
 // Lock order acquisition (canonical):
@@ -17,51 +23,218 @@ import (
 //  3. SnapshotCollection
 //  4. PublishedRepoCollection
 
-// GET /api/version
+type aptlyVersion struct {
+	// Aptly Version
+	Version string `json:"Version"`
+}
+
+// @Summary Aptly Version
+// @Description **Get aptly version**
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/version
+// @Description {"Version":"0.9~dev"}
+// @Description ```
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyVersion
+// @Router /api/version [get]
 func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
-// Periodically flushes CollectionFactory to free up memory used by collections,
-// flushing caches.
+type aptlyStatus struct {
+	// Aptly Status
+	Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"`
+}
+
+// @Summary Get Ready State
+// @Description **Get aptly ready state**
+// @Description
+// @Description Return aptly ready state:
+// @Description - `Aptly is ready` (HTTP 200)
+// @Description - `Aptly is unavailable` (HTTP 503)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus "Aptly is ready"
+// @Failure 503 {object} aptlyStatus "Aptly is unavailable"
+// @Router /api/ready [get]
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+	}
+}
+
+// @Summary Get Health State
+// @Description **Get aptly health state**
+// @Description
+// @Description Return aptly health state:
+// @Description - `Aptly is healthy` (HTTP 200)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus
+// @Router /api/healthy [get]
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
+}
+
+type dbRequestKind int
+
+const (
+	acquiredb dbRequestKind = iota
+	releasedb
+)
+
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
+var dbRequests chan dbRequest
+
+// Acquire database lock and release it when not needed anymore.
 //
-// Should be run in goroutine!
-func cacheFlusher() {
-	ticker := time.Tick(15 * time.Minute)
+// Should be run in a goroutine!
+func acquireDatabase() {
+	clients := 0
+	for request := range dbRequests {
+		var err error
 
-	for {
-		<-ticker
+		switch request.kind {
+		case acquiredb:
+			if clients == 0 {
+				err = context.ReOpenDatabase()
+			}
 
-		// lock everything to eliminate in-progress calls
-		r := context.CollectionFactory().RemoteRepoCollection()
-		r.Lock()
-		defer r.Unlock()
+			request.err <- err
 
-		l := context.CollectionFactory().LocalRepoCollection()
-		l.Lock()
-		defer l.Unlock()
+			if err == nil {
+				clients++
+			}
+		case releasedb:
+			clients--
+			if clients == 0 {
+				err = context.CloseDatabase()
+			} else {
+				err = nil
+			}
 
-		s := context.CollectionFactory().SnapshotCollection()
-		s.Lock()
-		defer s.Unlock()
+			request.err <- err
+		}
+	}
+}
 
-		p := context.CollectionFactory().PublishedRepoCollection()
-		p.Lock()
-		defer p.Unlock()
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
 
-		// all collections locked, flush them
-		context.CollectionFactory().Flush()
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Debug().Msg("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Debug().Msg("Executing task synchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
+		context.TaskList().DeleteTaskByID(task.ID)
+		if err != nil {
+			AbortWithJSONError(c, retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
 	}
 }
 
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.Fail(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -69,7 +242,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.Fail(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -86,20 +259,57 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			Source:            nil,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
+			return
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter(deb.FilterOptions{
+					Queries: []deb.PackageQuery{versionQ},
+				})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
 		list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
@@ -111,3 +321,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		c.JSON(200, list.Strings())
 	}
 }
+
+func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+	return c.AbortWithError(code, err)
+}

api/api_test.go

@@ -0,0 +1,175 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ApiSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+}
+
+var _ = Suite(&ApiSuite{})
+
+func createTestConfig() *os.File {
+	file, err := os.CreateTemp("", "aptly")
+	if err != nil {
+		return nil
+	}
+	jsonString, err := json.Marshal(gin.H{
+		"architectures":         []string{},
+		"enableMetricsEndpoint": true,
+	})
+	if err != nil {
+		return nil
+	}
+	file.Write(jsonString)
+	return file
+}
+
+func (s *ApiSuite) setupContext() error {
+	aptly.Version = "testVersion"
+	file := createTestConfig()
+	if nil == file {
+		return fmt.Errorf("unable to create the test configuration file")
+	}
+	s.configFile = file
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	if nil != err {
+		return err
+	}
+
+	s.context = context
+	s.router = Router(context)
+
+	return nil
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	err := s.setupContext()
+	c.Assert(err, IsNil)
+}
+
+func (s *ApiSuite) TearDownSuite(c *C) {
+	os.Remove(s.configFile.Name())
+	s.context.Shutdown()
+}
+
+func (s *ApiSuite) SetUpTest(c *C) {
+}
+
+func (s *ApiSuite) TearDownTest(c *C) {
+}
+
+func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w, nil
+}
+
+func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
+}
+
+func (s *ApiSuite) TestGetVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/version", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
+}
+
+func (s *ApiSuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *ApiSuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *ApiSuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
+}
+
+func (s *ApiSuite) TestRepoCreate(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name": "dummy",
+	})
+	c.Assert(err, IsNil)
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+}
+
+func (s *ApiSuite) TestTruthy(c *C) {
+	c.Check(truthy("no"), Equals, false)
+	c.Check(truthy("n"), Equals, false)
+	c.Check(truthy("off"), Equals, false)
+	c.Check(truthy("false"), Equals, false)
+	c.Check(truthy("0"), Equals, false)
+	c.Check(truthy(false), Equals, false)
+	c.Check(truthy(0), Equals, false)
+
+	c.Check(truthy("y"), Equals, true)
+	c.Check(truthy("yes"), Equals, true)
+	c.Check(truthy("t"), Equals, true)
+	c.Check(truthy("true"), Equals, true)
+	c.Check(truthy("1"), Equals, true)
+	c.Check(truthy(true), Equals, true)
+	c.Check(truthy(1), Equals, true)
+
+	c.Check(truthy(nil), Equals, false)
+
+	c.Check(truthy("foobar"), Equals, true)
+	c.Check(truthy(-1), Equals, true)
+	c.Check(truthy(gin.H{}), Equals, true)
+}

api/db.go

@@ -0,0 +1,188 @@
+package api
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary DB Cleanup
+// @Description **Cleanup Aptly DB**
+// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore.
+// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos.
+// @Tags Database
+// @Produce json
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/db/cleanup [post]
+func apiDbCleanup(c *gin.Context) {
+	resources := []string{string(task.AllResourcesKey)}
+	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var err error
+
+		collectionFactory := context.NewCollectionFactory()
+
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
+
+		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
+		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if e != nil {
+				return e
+			}
+
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+			if published.SourceKind != deb.SourceLocalRepo {
+				return nil
+			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
+
+			for _, component := range published.Components() {
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// ... and compare it to the list of all packages
+		out.Printf("Loading list of all packages...")
+		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+		// delete packages that are no longer referenced
+		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
+
+		// database can't err as collection factory already constructed
+		db, _ := context.Database()
+
+		if toDelete.Len() > 0 {
+			batch := db.CreateBatch()
+			toDelete.ForEach(func(ref []byte) error {
+				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				return nil
+			})
+
+			err = batch.Write()
+			if err != nil {
+				return nil, fmt.Errorf("unable to write to DB: %s", err)
+			}
+		}
+
+		// now, build a list of files that should be present in Repository (package pool)
+		out.Printf("Building list of files referenced by packages...")
+		referencedFiles := make([]string, 0, existingPackageRefs.Len())
+
+		err = existingPackageRefs.ForEach(func(key []byte) error {
+			pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+			if err2 != nil {
+				tail := ""
+				return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+			}
+			paths, err2 := pkg.FilepathList(context.PackagePool())
+			if err2 != nil {
+				return err2
+			}
+			referencedFiles = append(referencedFiles, paths...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sort.Strings(referencedFiles)
+
+		// build a list of files in the package pool
+		out.Printf("Building list of files in package pool...")
+		existingFiles, err := context.PackagePool().FilepathList(out)
+		if err != nil {
+			return nil, fmt.Errorf("unable to collect file paths: %s", err)
+		}
+
+		// find files which are in the pool but not referenced by packages
+		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+		// delete files that are no longer referenced
+		out.Printf("Deleting unreferenced files (%d)...", len(filesToDelete))
+
+		countFilesToDelete := len(filesToDelete)
+		taskDetail := struct {
+			TotalNumberOfPackagesToDelete     int
+			RemainingNumberOfPackagesToDelete int
+		}{
+			countFilesToDelete, countFilesToDelete,
+		}
+		detail.Store(taskDetail)
+
+		if countFilesToDelete > 0 {
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return nil, err
+				}
+
+				taskDetail.RemainingNumberOfPackagesToDelete--
+				detail.Store(taskDetail)
+				totalSize += size
+			}
+
+			out.Printf("Disk space freed: %s...", utils.HumanBytes(totalSize))
+		}
+
+		out.Printf("Compacting database...")
+		return nil, db.CompactDB()
+	})
+}

api/error.go

@@ -0,0 +1,5 @@
+package api
+
+type Error struct {
+	Error string `json:"error"`
+}

api/files.go

@@ -2,11 +2,15 @@ package api
 
 import (
 	"fmt"
-	"github.com/gin-gonic/gin"
 	"io"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
 func verifyPath(path string) bool {
@@ -23,27 +27,37 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.Fail(400, fmt.Errorf("wrong dir"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
 	return true
 }
 
-// GET /files
+// @Summary List Directories
+// @Description **Get list of upload directories**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description  $ curl http://localhost:8080/api/files
+// @Description  ["aptly-0.9"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Success 200 {array} string "List of files"
+// @Router /api/files [get]
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -52,30 +66,50 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.Fail(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, list)
 }
 
-// POST /files/:dir/
+// @Summary Upload Files
+// @Description **Upload files to a directory**
+// @Description
+// @Description - one or more files can be uploaded
+// @Description - existing uploaded are overwritten
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Accept multipart/form-data
+// @Param dir path string true "Directory to upload files to. Created if does not exist"
+// @Param files formData file true "Files to upload"
+// @Produce json
+// @Success 200 {array} string "list of uploaded files"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [post]
 func apiFilesUpload(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.Fail(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -85,7 +119,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer src.Close()
@@ -93,14 +127,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -108,20 +142,35 @@ func apiFilesUpload(c *gin.Context) {
 		}
 	}
 
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
-
 }
 
-// GET /files/:dir
+// @Summary List Files
+// @Description **Show uploaded files in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to list"
+// @Success 200 {array} string "Files found in directory"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [get]
 func apiFilesListFiles(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
 	list := []string{}
-	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	listLock := &sync.Mutex{}
+	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -130,6 +179,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil
@@ -137,9 +188,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.Fail(404, err)
+			AbortWithJSONError(c, 404, err)
 		} else {
-			c.Fail(500, err)
+			AbortWithJSONError(c, 500, err)
 		}
 		return
 	}
@@ -147,36 +198,66 @@ func apiFilesListFiles(c *gin.Context) {
 	c.JSON(200, list)
 }
 
-// DELETE /files/:dir
+// @Summary Delete Directory
+// @Description **Delete upload directory and uploaded files within**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [delete]
 func apiFilesDeleteDir(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, gin.H{})
 }
 
-// DELETE /files/:dir/:name
+// @Summary Delete File
+// @Description **Delete a uploaded file in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to delete from"
+// @Param name path string true "File to delete"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir}/{name} [delete]
 func apiFilesDeleteFile(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	if !verifyPath(c.Params.ByName("name")) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+	dir := utils.SanitizePath(c.Params.ByName("dir"))
+	name := utils.SanitizePath(c.Params.ByName("name"))
+	if !verifyPath(name) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.Fail(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 	}

api/gpg.go

@@ -0,0 +1,102 @@
+package api
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+type gpgAddKeyParams struct {
+	// Keyserver, when downloading GpgKeyIDs
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// GpgKeyIDs to download from Keyserver, comma separated list
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
+	// Armored gpg public ket, instead of downloading from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+	// Keyring for adding the keys (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+}
+
+// @Summary Add GPG Keys
+// @Description **Adds GPG keys to aptly keyring**
+// @Description
+// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/gpg [post]
+func apiGPGAddKey(c *gin.Context) {
+	b := gpgAddKeyParams{}
+	if c.Bind(&b) != nil {
+		return
+	}
+	b.Keyserver = utils.SanitizePath(b.Keyserver)
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
+	// b.Keyring can be an absolute path
+
+	var err error
+	args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+	args = append(args, "--keyring", keyring)
+	if len(b.Keyserver) > 0 {
+		args = append(args, "--keyserver", b.Keyserver)
+	}
+	if len(b.GpgKeyArmor) > 0 {
+		var tempdir string
+		tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+		defer os.RemoveAll(tempdir)
+
+		keypath := filepath.Join(tempdir, "key")
+		keyfile, e := os.Create(keypath)
+		if e != nil {
+			AbortWithJSONError(c, 400, e)
+			return
+		}
+		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
+			AbortWithJSONError(c, 400, e)
+		}
+		args = append(args, "--import", keypath)
+
+	}
+	if len(b.GpgKeyID) > 0 {
+		keys := strings.Fields(b.GpgKeyID)
+		args = append(args, "--recv-keys")
+		args = append(args, keys...)
+	}
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	// it might happened that we have a situation with an erroneous
+	// gpg command (e.g. when GpgKeyID and GpgKeyArmor is set).
+	// there is no error handling for such as gpg will do this for us
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		c.JSON(400, string(out))
+		return
+	}
+
+	c.JSON(200, string(out))
+}

api/graph.go

@@ -3,15 +3,36 @@ package api
 import (
 	"bytes"
 	"fmt"
-	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/deb"
 	"io"
 	"mime"
 	"os"
 	"os/exec"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
 )
 
-// GET /api/graph.:ext
+// @Summary Graph Output
+// @Description **Generate dependency graph**
+// @Description
+// @Description Command graph generates graph of dependencies:
+// @Description
+// @Description * between snapshots and mirrors (what mirror was used to create each snapshot)
+// @Description * between snapshots and local repos (what local repo was used to create snapshot)
+// @Description * between snapshots (pulling, merging, etc.)
+// @Description * between snapshots, local repos and published repositories (how snapshots were published).
+// @Description
+// @Description Graph is rendered to PNG file using graphviz package.
+// @Description
+// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical`
+// @Tags Status
+// @Produce image/png, image/svg+xml
+// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg."
+// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout."
+// @Success 200 {object} []byte "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/graph.{ext} [get]
 func apiGraph(c *gin.Context) {
 	var (
 		err    error
@@ -19,19 +40,10 @@ func apiGraph(c *gin.Context) {
 	)
 
 	ext := c.Params.ByName("ext")
+	layout := c.Request.URL.Query().Get("layout")
+	factory := context.NewCollectionFactory()
 
-	factory := context.CollectionFactory()
-
-	factory.RemoteRepoCollection().RLock()
-	defer factory.RemoteRepoCollection().RUnlock()
-	factory.LocalRepoCollection().RLock()
-	defer factory.LocalRepoCollection().RUnlock()
-	factory.SnapshotCollection().RLock()
-	defer factory.SnapshotCollection().RUnlock()
-	factory.PublishedRepoCollection().RLock()
-	defer factory.PublishedRepoCollection().RUnlock()
-
-	graph, err := deb.BuildGraph(factory)
+	graph, err := deb.BuildGraph(factory, layout)
 	if err != nil {
 		c.JSON(500, err)
 		return
@@ -39,30 +51,37 @@ func apiGraph(c *gin.Context) {
 
 	buf := bytes.NewBufferString(graph.String())
 
+	if ext == "dot" || ext == "gv" {
+		// If the raw dot data is requested, return it as string.
+		// This allows client-side rendering rather than server-side.
+		c.String(200, buf.String())
+		return
+	}
+
 	command := exec.Command("dot", "-T"+ext)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/metrics.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+	apiFilesUploadedCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_files_uploaded_total",
+			Help: "Total number of uploaded files labeled by upload directory.",
+		},
+		[]string{"directory"},
+	)
+	apiReposPackageCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_repos_package_count",
+			Help: "Current number of published packages labeled by source, distribution and component.",
+		},
+		[]string{"source", "distribution", "component"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
+
+func countPackagesByRepos() {
+	err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if err != nil {
+			msg := fmt.Sprintf(
+				"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
+				err, repo.StoragePrefix(), repo.Distribution, repo.Components())
+			log.Warn().Msg(msg)
+			return err
+		}
+
+		components := repo.Components()
+		for _, c := range components {
+			count := float64(len(repo.RefList(c).Refs))
+			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
+		log.Warn().Msg(msg)
+	}
+}

api/middleware.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog/log"
+)
+
+// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
+// See https://prometheus.io/docs/practices/naming/#labels
+func getBasePath(c *gin.Context) string {
+	segment0, err := getURLSegment(c.Request.URL.Path, 0)
+	if err != nil {
+		return "/"
+	}
+	segment1, err := getURLSegment(c.Request.URL.Path, 1)
+	if err != nil {
+		return *segment0
+	}
+
+	return *segment0 + *segment1
+}
+
+func getURLSegment(url string, idx int) (*string, error) {
+	urlSegments := strings.Split(url, "/")
+	// Remove segment at index 0 because it's an empty string
+	urlSegments = urlSegments[1:cap(urlSegments)]
+
+	if len(urlSegments) <= idx {
+		return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
+	}
+
+	segmentAtIndex := urlSegments[idx]
+	s := fmt.Sprintf("/%s", segmentAtIndex)
+	return &s, nil
+}
+
+func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		g.WithLabelValues(c.Request.Method, pathFunc(c)).Inc()
+		defer g.WithLabelValues(c.Request.Method, pathFunc(c)).Dec()
+		c.Next()
+	}
+}
+
+func instrumentHandlerCounter(counter *prometheus.CounterVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		counter.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Inc()
+	}
+}
+
+func instrumentHandlerRequestSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(float64(c.Request.ContentLength))
+	}
+}
+
+func instrumentHandlerResponseSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		var responseSize = math.Max(float64(c.Writer.Size()), 0)
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(responseSize)
+	}
+}
+
+func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		now := time.Now()
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
+	}
+}
+
+// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
+// logs that include error messages if there are any.
+func JSONLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
+
+		// Process request
+		c.Next()
+
+		ts := time.Now()
+		if raw != "" {
+			path = path + "?" + raw
+		}
+
+		errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
+		l := log.With().Str("remote", c.ClientIP()).Logger().
+			With().Str("method", c.Request.Method).Logger().
+			With().Str("path", path).Logger().
+			With().Str("protocol", c.Request.Proto).Logger().
+			With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
+			With().Str("latency", ts.Sub(start).String()).Logger().
+			With().Str("agent", c.Request.UserAgent()).Logger()
+
+		if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
+			l.Warn().Msg(errorMessage)
+		} else if c.Writer.Status() >= 500 {
+			l.Error().Msg(errorMessage)
+		} else {
+			l.Info().Msg(errorMessage)
+		}
+	}
+}

api/middleware_test.go

@@ -0,0 +1,255 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MiddlewareSuite struct {
+	router    http.Handler
+	context   *gin.Context
+	logReader *os.File
+	logWriter *os.File
+}
+
+var _ = Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) SetUpTest(c *C) {
+	r, w, err := os.Pipe()
+	c.Assert(err, IsNil)
+
+	utils.SetupJSONLogger("debug", w)
+	mw := JSONLogger()
+
+	router := gin.New()
+	router.UseRawPath = true
+	router.Use(mw)
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	root := router.Group("/api")
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	root.GET("/ready", apiReady(isReady))
+	root.GET("/healthy", apiHealthy)
+
+	s.router = router
+	s.logReader = r
+	s.logWriter = w
+}
+
+func (s *MiddlewareSuite) TearDownTest(c *C) {
+	s.router = nil
+	s.context = nil
+	s.logReader = nil
+	s.logWriter = nil
+}
+
+func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
+	recorder := httptest.NewRecorder()
+	s.context, _ = gin.CreateTestContext(recorder)
+	req, _ := http.NewRequestWithContext(s.context, method, url, body)
+	s.context.Request = req
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "warn")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["method"]; ok {
+		c.Check(val, Equals, "GET")
+	} else {
+		c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["path"]; ok {
+		c.Check(val, Equals, "/")
+	} else {
+		c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["protocol"]; ok {
+		c.Check(val, Equals, "HTTP/1.1")
+	} else {
+		c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["code"]; ok {
+		c.Check(val, Equals, "404")
+	} else {
+		c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["remote"]; !ok {
+		c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["latency"]; !ok {
+		c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["agent"]; !ok {
+		c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["time"]; !ok {
+		c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "error")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	fmt.Println(capturedOutput)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestGetBasePath(c *C) {
+	s.HTTPRequest(http.MethodGet, "", nil)
+	path := getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/api", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api")
+
+	s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api/repos")
+}
+
+func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
+	url := "/"
+	segment, err := getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api/repos/testRepo"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	segment, err = getURLSegment(url, 1)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/repos")
+}

api/mirror.go

@@ -0,0 +1,665 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
+)
+
+func getVerifier(keyRings []string) (pgp.Verifier, error) {
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring(false)
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+// @Summary List Mirrors
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {array} deb.RemoteRepo
+// @Router /api/mirrors [get]
+func apiMirrorsList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	result := []*deb.RemoteRepo{}
+	collection.ForEach(func(repo *deb.RemoteRepo) error {
+		result = append(result, repo)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
+// @Summary Create Mirror
+// @Description **Create a mirror of a remote repository**
+// @Tags Mirrors
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/mirrors [post]
+func apiMirrorsCreate(c *gin.Context) {
+	var err error
+	var b mirrorCreateParams
+
+	b.DownloadSources = context.Config().DownloadSourcePackages
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+	b.Architectures = context.ArchitecturesList()
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
+		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+	}
+
+	if b.Filter != "" {
+		_, err = query.Parse(b.Filter)
+		if err != nil {
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
+			return
+		}
+	}
+
+	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
+		return
+	}
+
+	repo.Filter = b.Filter
+	repo.FilterWithDeps = b.FilterWithDeps
+	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.SkipArchitectureCheck = b.SkipArchitectureCheck
+	repo.DownloadSources = b.DownloadSources
+	repo.DownloadUdebs = b.DownloadUdebs
+
+	verifier, err := getVerifier(b.Keyrings)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	downloader := context.NewDownloader(nil)
+	err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
+		return
+	}
+
+	err = collection.Add(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// @Summary Delete Mirror
+// @Description **Delete a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param force query int true "force: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
+// @Failure 500 {object} Error "Unable to delete"
+// @Router /api/mirrors/{name} [delete]
+func apiMirrorsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	mirrorCollection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	repo, err := mirrorCollection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+			}
+		}
+
+		err = mirrorCollection.Drop(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}
+
+// @Summary Get Mirror Info
+// @Description **Get mirror information by name**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [get]
+func apiMirrorsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	c.JSON(200, repo)
+}
+
+// @Summary List Mirror Packages
+// @Description **Get a list of packages from a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Produce json
+// @Success 200 {array} deb.Package "List of Packages"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name}/packages [get]
+func apiMirrorsPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	if repo.LastDownloadDate.IsZero() {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		return
+	}
+
+	reflist := repo.RefList()
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}
+
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Url of the archive to mirror
+	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
+	// Package query that is applied to mirror packages
+	Filter string `               json:"Filter"                 example:"xserver-xorg"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `      json:"Architectures"          example:"amd64"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `         json:"Components"             example:"main"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `         json:"FilterWithDeps"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `        json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `          json:"DownloadUdebs"`
+	// Set "true" to skip checking if the given components are in the Release file
+	SkipComponentCheck bool `     json:"SkipComponentCheck"`
+	// Set "true" to skip checking if the given architectures are in the Release file
+	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+}
+
+// @Summary Update Mirror
+// @Description **Update Mirror and download packages**
+// @Tags Mirrors
+// @Param name path string true "mirror name to update"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [put]
+func apiMirrorsUpdate(c *gin.Context) {
+	var (
+		err    error
+		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
+	)
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	remote, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	b.Name = remote.Name
+	b.DownloadUdebs = remote.DownloadUdebs
+	b.DownloadSources = remote.DownloadSources
+	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
+	b.FilterWithDeps = remote.FilterWithDeps
+	b.Filter = remote.Filter
+	b.Architectures = remote.Architectures
+	b.Components = remote.Components
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+
+	log.Info().Msgf("%s: Starting mirror update", b.Name)
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Name != remote.Name {
+		_, err = collection.ByName(b.Name)
+		if err == nil {
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			return
+		}
+	}
+
+	if b.DownloadUdebs != remote.DownloadUdebs {
+		if remote.IsFlat() && b.DownloadUdebs {
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			return
+		}
+	}
+
+	if b.ArchiveURL != "" {
+		remote.SetArchiveRoot(b.ArchiveURL)
+	}
+
+	remote.Name = b.Name
+	remote.DownloadUdebs = b.DownloadUdebs
+	remote.DownloadSources = b.DownloadSources
+	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
+	remote.FilterWithDeps = b.FilterWithDeps
+	remote.Filter = b.Filter
+	remote.Architectures = b.Architectures
+	remote.Components = b.Components
+
+	verifier, err := getVerifier(b.Keyrings)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	resources := []string{string(remote.Key())}
+	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+		downloader := context.NewDownloader(out)
+		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if !b.ForceUpdate {
+			err = remote.CheckLock()
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if remote.Filter != "" {
+			var filterQuery deb.PackageQuery
+
+			filterQuery, err = query.Parse(remote.Filter)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			_, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		defer func() {
+			// on any interruption, unlock the mirror
+			e := context.ReOpenDatabase()
+			if e == nil {
+				remote.MarkAsIdle()
+				collection.Update(remote)
+			}
+		}()
+
+		remote.MarkAsUpdating()
+		err = collection.Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		context.GoContextHandleSignals()
+
+		count := len(queue)
+		taskDetail := struct {
+			TotalDownloadSize         int64
+			RemainingDownloadSize     int64
+			TotalNumberOfPackages     int
+			RemainingNumberOfPackages int
+		}{
+			downloadSize, downloadSize, count, count,
+		}
+		detail.Store(taskDetail)
+
+		downloadQueue := make(chan int)
+		taskFinished := make(chan *deb.PackageDownloadTask)
+
+		var (
+			errors  []string
+			errLock sync.Mutex
+		)
+
+		pushError := func(err error) {
+			errLock.Lock()
+			errors = append(errors, err.Error())
+			errLock.Unlock()
+		}
+
+		go func() {
+			for idx := range queue {
+				select {
+				case downloadQueue <- idx:
+				case <-context.Done():
+					return
+				}
+			}
+
+			close(downloadQueue)
+		}()
+
+		// update of task details need to be done in order
+		go func() {
+			for {
+				task, ok := <-taskFinished
+				if !ok {
+					return
+				}
+
+				taskDetail.RemainingDownloadSize -= task.File.Checksums.Size
+				taskDetail.RemainingNumberOfPackages--
+				detail.Store(taskDetail)
+			}
+		}()
+
+		log.Info().Msgf("%s: Spawning background processes...", b.Name)
+		var wg sync.WaitGroup
+		for i := 0; i < context.Config().DownloadConcurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case idx, ok := <-downloadQueue:
+						if !ok {
+							return
+						}
+
+						task := &queue[idx]
+
+						var e error
+
+						// provision download location
+						if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+							task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+						} else {
+							var file *os.File
+							file, e = os.CreateTemp("", task.File.Filename)
+							if e == nil {
+								task.TempDownPath = file.Name()
+								file.Close()
+							}
+						}
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// download file...
+						e = context.Downloader().DownloadWithChecksum(
+							context,
+							remote.PackageURL(task.File.DownloadURL()).String(),
+							task.TempDownPath,
+							&task.File.Checksums,
+							b.IgnoreChecksums)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// and import it back to the pool
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						if err != nil {
+							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+							pushError(err)
+							continue
+						}
+
+						// update "attached" files if any
+						for _, additionalAtask := range task.Additional {
+							additionalAtask.File.PoolPath = task.File.PoolPath
+							additionalAtask.File.Checksums = task.File.Checksums
+						}
+
+						task.Done = true
+						taskFinished <- task
+					case <-context.Done():
+						return
+					}
+
+				}
+			}()
+		}
+
+		// Wait for all download goroutines to finish
+		log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
+		wg.Wait()
+		log.Info().Msgf("%s: Background processes finished", b.Name)
+		close(taskFinished)
+
+		defer func() {
+			for _, task := range queue {
+				if task.TempDownPath == "" {
+					continue
+				}
+
+				if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+					fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+				}
+			}
+		}()
+
+		select {
+		case <-context.Done():
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+
+		if len(errors) > 0 {
+			log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+		}
+
+		log.Info().Msgf("%s: Finalizing download...", b.Name)
+		remote.FinalizeDownload(collectionFactory, out)
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		log.Info().Msgf("%s: Mirror updated successfully", b.Name)
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}

api/mirror_test.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MirrorSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&MirrorSuite{})
+
+func (s *MirrorSuite) TestGetMirrors(c *C) {
+	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}
+
+func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
+	response, _ := s.HTTPRequest("DELETE", "/api/mirrors/does-not-exist", nil)
+	c.Check(response.Code, Equals, 404)
+	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
+}
+
+func (s *MirrorSuite) TestCreateMirror(c *C) {
+	c.ExpectFailure("Need to mock downloads")
+	body, err := json.Marshal(gin.H{
+		"Name":       "dummy",
+		"ArchiveURL": "foobar",
+	})
+	c.Assert(err, IsNil)
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Equals, "")
+}

api/packages.go

@@ -1,16 +1,41 @@
 package api
 
 import (
+	_ "github.com/aptly-dev/aptly/deb" // for swagger
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/packages/:key
+// @Summary Get Package Info
+// @Description **Show information about package by package key**
+// @Description Package keys could be obtained from various GET .../packages APIs.
+// @Tags Packages
+// @Produce json
+// @Param key path string true "package key (unique package identifier)"
+// @Success 200 {object} deb.Package "OK"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/packages/{key} [get]
 func apiPackagesShow(c *gin.Context) {
-	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	collectionFactory := context.NewCollectionFactory()
+	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.Fail(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, p)
 }
+
+// @Summary List Packages
+// @Description **Get list of packages**
+// @Tags Packages
+// @Consume  json
+// @Produce  json
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Success 200 {array} string "List of packages"
+// @Router /api/packages [get]
+func apiPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PackageCollection()
+	showPackages(c, collection.AllPackageRefs(), collectionFactory)
+}

api/packages_test.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type PackagesSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&PackagesSuite{})
+
+func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}

api/router.go

@@ -1,81 +1,237 @@
 package api
 
 import (
-	"github.com/gin-gonic/gin"
-	ctx "github.com/smira/aptly/context"
 	"net/http"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog/log"
+
+	"github.com/aptly-dev/aptly/docs"
+	swaggerFiles "github.com/swaggo/files"
+	ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+func apiMetricsGet() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		countPackagesByRepos()
+		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
+	}
+}
+
+func redirectSwagger(c *gin.Context) {
+	if c.Request.URL.Path == "/docs/index.html" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs/" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	c.Next()
+}
+
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
+	if aptly.EnableDebug {
+		gin.SetMode(gin.DebugMode)
+	} else {
+		gin.SetMode(gin.ReleaseMode)
+	}
+
+	router := gin.New()
 	context = c
 
-	go cacheFlusher()
+	router.UseRawPath = true
 
-	router := gin.Default()
-	router.Use(gin.ErrorLogger())
+	if c.Config().LogFormat == "json" {
+		c.StructuredLogging(true)
+		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
+		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
+		router.Use(JSONLogger())
+	} else {
+		c.StructuredLogging(false)
+		utils.SetupDefaultLogger(c.Config().LogLevel)
+		router.Use(gin.Logger())
+	}
 
-	root := router.Group("/api")
+	router.Use(gin.Recovery(), gin.ErrorLogger())
 
-	{
-		root.GET("/version", apiVersion)
+	if c.Config().EnableSwaggerEndpoint {
+		router.GET("docs.html", func(c *gin.Context) {
+			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+		})
+		router.Use(redirectSwagger)
+		url := ginSwagger.URL("/docs/doc.json")
+		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	}
+
+	if c.Config().EnableMetricsEndpoint {
+		MetricsCollectorRegistrar.Register(router)
+	}
+
+	if c.Config().ServeInAPIMode {
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
+		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
+	}
+
+	api := router.Group("/api")
+	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
+		// We use a goroutine to count the number of
+		// concurrent requests. When no more requests are
+		// running, we close the database to free the lock.
+		dbRequests = make(chan dbRequest)
+
+		go acquireDatabase()
+
+		api.Use(func(c *gin.Context) {
+			var err error
+
+			errCh := make(chan error)
+			dbRequests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
+			if err != nil {
+				AbortWithJSONError(c, 500, err)
+				return
+			}
+
+			defer func() {
+				dbRequests <- dbRequest{releasedb, errCh}
+				err = <-errCh
+				if err != nil {
+					AbortWithJSONError(c, 500, err)
+				}
+			}()
+
+			c.Next()
+		})
 	}
 
 	{
-		root.GET("/repos", apiReposList)
-		root.POST("/repos", apiReposCreate)
-		root.GET("/repos/:name", apiReposShow)
-		root.PUT("/repos/:name", apiReposEdit)
-		root.DELETE("/repos/:name", apiReposDrop)
+		if c.Config().EnableMetricsEndpoint {
+			api.GET("/metrics", apiMetricsGet())
+		}
+		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
 
-		root.GET("/repos/:name/packages", apiReposPackagesShow)
-		root.POST("/repos/:name/packages", apiReposPackagesAdd)
-		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
-
-		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
-		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
-
-		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		api.GET("/ready", apiReady(isReady))
+		api.GET("/healthy", apiHealthy)
 	}
 
 	{
-		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+		api.GET("/repos", apiReposList)
+		api.POST("/repos", apiReposCreate)
+		api.GET("/repos/:name", apiReposShow)
+		api.PUT("/repos/:name", apiReposEdit)
+		api.DELETE("/repos/:name", apiReposDrop)
+
+		api.GET("/repos/:name/packages", apiReposPackagesShow)
+		api.POST("/repos/:name/packages", apiReposPackagesAdd)
+		api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
+
+		api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
+		api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 
 	{
-		root.GET("/files", apiFilesListDirs)
-		root.POST("/files/:dir", apiFilesUpload)
-		root.GET("/files/:dir", apiFilesListFiles)
-		root.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+		api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
 	{
-		root.GET("/publish", apiPublishList)
-		root.POST("/publish", apiPublishRepoOrSnapshot)
-		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
-		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
-		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.GET("/mirrors", apiMirrorsList)
+		api.GET("/mirrors/:name", apiMirrorsShow)
+		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		api.POST("/mirrors", apiMirrorsCreate)
+		api.PUT("/mirrors/:name", apiMirrorsUpdate)
+		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
-		root.GET("/snapshots", apiSnapshotsList)
-		root.POST("/snapshots", apiSnapshotsCreate)
-		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
-		root.GET("/snapshots/:name", apiSnapshotsShow)
-		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
-		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
-		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/gpg/key", apiGPGAddKey)
 	}
 
 	{
-		root.GET("/packages/:key", apiPackagesShow)
+		api.GET("/s3", apiS3List)
 	}
 
 	{
-		root.GET("/graph.:ext", apiGraph)
+		api.GET("/files", apiFilesListDirs)
+		api.POST("/files/:dir", apiFilesUpload)
+		api.GET("/files/:dir", apiFilesListFiles)
+		api.DELETE("/files/:dir", apiFilesDeleteDir)
+		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		api.GET("/publish", apiPublishList)
+		api.GET("/publish/:prefix/:distribution", apiPublishShow)
+		api.POST("/publish", apiPublishRepoOrSnapshot)
+		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource)
+		api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges)
+		api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources)
+		api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges)
+		api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource)
+		api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource)
+		api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate)
+	}
+
+	{
+		api.GET("/snapshots", apiSnapshotsList)
+		api.POST("/snapshots", apiSnapshotsCreate)
+		api.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.GET("/snapshots/:name", apiSnapshotsShow)
+		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
+	}
+
+	{
+		api.GET("/packages/:key", apiPackagesShow)
+		api.GET("/packages", apiPackages)
+	}
+
+	{
+		api.GET("/graph.:ext", apiGraph)
+	}
+	{
+		api.POST("/db/cleanup", apiDbCleanup)
+	}
+	{
+		api.GET("/tasks", apiTasksList)
+		api.POST("/tasks-clear", apiTasksClear)
+		api.GET("/tasks-wait", apiTasksWait)
+		api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		api.GET("/tasks/:id/output", apiTasksOutputShow)
+		api.GET("/tasks/:id/detail", apiTasksDetailShow)
+		api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		api.GET("/tasks/:id", apiTasksShow)
+		api.DELETE("/tasks/:id", apiTasksDelete)
 	}
 
 	return router

api/s3.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary S3 buckets
+// @Description **Get list of S3 buckets**
+// @Description
+// @Description List configured S3 buckets.
+// @Tags Status
+// @Produce json
+// @Success 200 {array} string "List of S3 buckets"
+// @Router /api/s3 [get]
+func apiS3List(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().S3PublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/storage.go

@@ -0,0 +1,45 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Description
+// @Description Units in MiB.
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

api/task.go

@@ -0,0 +1,203 @@
+package api
+
+import (
+	"strconv"
+
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary List Tasks
+// @Description **Get list of available tasks. Each task is returned as in “show” API**
+// @Tags Tasks
+// @Produce json
+// @Success 200 {array} task.Task
+// @Router /api/tasks [get]
+func apiTasksList(c *gin.Context) {
+	list := context.TaskList()
+	c.JSON(200, list.GetTasks())
+}
+
+// @Summary Clear Tasks
+// @Description **Removes finished and failed tasks from internal task list**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-clear [post]
+func apiTasksClear(c *gin.Context) {
+	list := context.TaskList()
+	list.Clear()
+	c.JSON(200, gin.H{})
+}
+
+// @Summary Wait for all Tasks
+// @Description **Waits for and returns when all running tasks are complete**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-wait [get]
+func apiTasksWait(c *gin.Context) {
+	list := context.TaskList()
+	list.Wait()
+	c.JSON(200, gin.H{})
+}
+
+// @Summary Wait for Task
+// @Description **Waits for and returns when given Task ID is complete**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 400 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/wait [get]
+func apiTasksWaitForTaskByID(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	task, err := list.WaitForTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// @Summary Get Task Info
+// @Description **Return task information for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id} [get]
+func apiTasksShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var task task.Task
+	task, err = list.GetTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// @Summary Get Task Output
+// @Description **Return task output for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task output"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/output [get]
+func apiTasksOutputShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var output string
+	output, err = list.GetTaskOutputByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// @Summary Get Task Details
+// @Description **Return task detail for a given ID**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task detail"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/detail [get]
+func apiTasksDetailShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var detail interface{}
+	detail, err = list.GetTaskDetailByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, detail)
+}
+
+// @Summary Get Task Return Value
+// @Description **Return task return value (status code) by given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/tasks/{id}/return_value [get]
+func apiTasksReturnValueShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	output, err := list.GetTaskReturnValueByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// @Summary Delete Task
+// @Description **Delete completed task by given ID. Does not stop task execution**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 400 {object} Error "Task in progress or not found"
+// @Router /api/tasks/{id} [delete]
+func apiTasksDelete(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var delTask task.Task
+	delTask, err = list.DeleteTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	c.JSON(200, delTask)
+}

aptly/conf.go

@@ -0,0 +1,4 @@
+package aptly
+
+// Default aptly.conf (filled in at link time)
+var AptlyConf []byte

aptly/const.go

@@ -0,0 +1,3 @@
+package aptly
+
+var DistributionFocal = "focal"

aptly/interfaces.go

@@ -3,24 +3,62 @@
 package aptly
 
 import (
-	"github.com/smira/aptly/utils"
+	"context"
 	"io"
+	"os"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/utils"
 )
 
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
 // PackagePool is asbtraction of package pool storage.
 //
 // PackagePool stores all the package files, deduplicating them.
 type PackagePool interface {
-	// Path returns full path to package file in pool given any name and hash of file contents
-	Path(filename string, hashMD5 string) (string, error)
-	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
-	RelativePath(filename string, hashMD5 string) (string, error)
+	// Verify checks whether file exists in the pool and fills back checksum info
+	//
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Size returns the size of the given file in bytes.
+	Size(path string) (size int64, err error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
 	FilepathList(progress Progress) ([]string, error)
 	// Remove deletes file in package pool returns its size
 	Remove(path string) (size int64, err error)
-	// Import copies file into package pool
-	Import(path string, hashMD5 string) error
+}
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
 }
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
@@ -34,15 +72,23 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
-// LocalPublishedStorage is published storage on local filesystem
-type LocalPublishedStorage interface {
+// FileSystemPublishedStorage is published storage on filesystem
+type FileSystemPublishedStorage interface {
 	// PublicPath returns root of public part
 	PublicPath() string
 }
@@ -53,6 +99,36 @@ type PublishedStorageProvider interface {
 	GetPublishedStorage(name string) PublishedStorage
 }
 
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -64,7 +140,7 @@ type Progress interface {
 	// Flush returns when all queued messages are sent
 	Flush()
 	// InitBar starts progressbar for count bytes or count items
-	InitBar(count int64, isBytes bool)
+	InitBar(count int64, isBytes bool, barType BarType)
 	// ShutdownBar stops progress bar and hides it
 	ShutdownBar()
 	// AddBar increments progress for progress bar
@@ -75,23 +151,29 @@ type Progress interface {
 	Printf(msg string, a ...interface{})
 	// ColoredPrintf does printf in colored way + newline
 	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
 }
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string, result chan<- error)
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
-	// Pause pauses task processing
-	Pause()
-	// Resume resumes task processing
-	Resume()
-	// Shutdown stops downloader after current tasks are finished,
-	// but doesn't process rest of queue
-	Shutdown()
-	// Abort stops downloader without waiting for shutdown
-	Abort()
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
+}
+
+// ChecksumStorageProvider creates ChecksumStorage based on DB
+type ChecksumStorageProvider func(db database.ReaderWriter) ChecksumStorage
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
 }

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
-// Version of aptly
-const Version = "0.9.1"
+// Version of aptly (filled in at link time)
+var Version string
 
-// Enable debugging features?
+// EnableDebug triggers some debugging features
 const EnableDebug = false

azure/azure.go

@@ -0,0 +1,136 @@
+package azure
+
+// Package azure handles publishing to Azure Storage
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+func isBlobNotFound(err error) bool {
+	var respErr *azcore.ResponseError
+	if errors.As(err, &respErr) {
+		return respErr.StatusCode == 404 // BlobNotFound
+	}
+	return false
+}
+
+type azContext struct {
+	client    *azblob.Client
+	container string
+	prefix    string
+}
+
+func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
+	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+	}
+
+	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	result := &azContext{
+		client:    serviceClient,
+		container: container,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+func (az *azContext) blobPath(path string) string {
+	return filepath.Join(az.prefix, path)
+}
+
+func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(az.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	ctx := context.Background()
+	maxResults := int32(1)
+	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
+		Prefix:     &prefix,
+		MaxResults: &maxResults,
+		Include:    azblob.ListBlobsInclude{Metadata: true},
+	})
+
+	// Iterate over each page
+	for pager.More() {
+		page, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
+		}
+
+		for _, blob := range page.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, *blob.Name)
+			} else {
+				name := *blob.Name
+				paths = append(paths, name[len(prefix):])
+			}
+			b := *blob
+			md5 := b.Properties.ContentMD5
+			md5s = append(md5s, fmt.Sprintf("%x", md5))
+
+		}
+		if progress != nil {
+			time.Sleep(time.Duration(500) * time.Millisecond)
+			progress.AddBar(1)
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
+	uploadOptions := &azblob.UploadFileOptions{
+		BlockSize:   4 * 1024 * 1024,
+		Concurrency: 8,
+	}
+
+	path := az.blobPath(blobName)
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
+			BlobContentMD5: decodedMD5,
+		}
+	}
+
+	var err error
+	if file, ok := source.(*os.File); ok {
+		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
+	}
+
+	return err
+}
+
+// String
+func (az *azContext) String() string {
+	return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
+}

azure/azure_test.go

@@ -0,0 +1,12 @@
+package azure
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

azure/package_pool.go

@@ -0,0 +1,215 @@
+package azure
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+type PackagePool struct {
+	az *azContext
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates published storage from Azure storage credentials
+func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagePool{az: azctx}, nil
+}
+
+// String
+func (pool *PackagePool) String() string {
+	return pool.az.String()
+}
+
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
+	hash := checksums.SHA256
+	// Use the same path as the file pool, for compat reasons.
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
+}
+
+func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+	targetChecksums, err := checksumStorage.Get(poolPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		if err != nil {
+			if isBlobNotFound(err) {
+				return nil, nil
+			}
+
+			return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
+		}
+
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return targetChecksums, nil
+}
+
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	if progress != nil {
+		progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
+		defer progress.ShutdownBar()
+	}
+
+	paths, _, err := pool.az.internalFilelist("", progress)
+	return paths, err
+}
+
+func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
+	return "", errors.New("Azure package pool does not support legacy paths")
+}
+
+func (pool *PackagePool) Size(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	temp, err := os.CreateTemp("", "blob-download")
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+	}
+	defer os.Remove(temp.Name())
+
+	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+	}
+
+	return temp, nil
+}
+
+func (pool *PackagePool) Remove(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		var err error
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	path := pool.buildPoolPath(basename, checksums)
+	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
+	if err != nil {
+		return "", err
+	} else if targetChecksums != nil {
+		// target already exists
+		*checksums = *targetChecksums
+		return path, nil
+	}
+
+	source, err := os.Open(srcPath)
+	if err != nil {
+		return "", err
+	}
+	defer source.Close()
+
+	err = pool.az.putFile(path, source, checksums.MD5)
+	if err != nil {
+		return "", err
+	}
+
+	if !checksums.Complete() {
+		// need full checksums here
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	err = checksumStorage.Update(path, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	if poolPath == "" {
+		if checksums.SHA256 != "" {
+			poolPath = pool.buildPoolPath(basename, checksums)
+		} else {
+			// No checksums or pool path, so no idea what file to look for.
+			return "", false, nil
+		}
+	}
+
+	size, err := pool.Size(poolPath)
+	if err != nil {
+		return "", false, err
+	} else if size != checksums.Size {
+		return "", false, nil
+	}
+
+	targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
+	if err != nil {
+		return "", false, err
+	} else if targetChecksums == nil {
+		return "", false, nil
+	}
+
+	if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+		checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+		// wrong file?
+		return "", false, nil
+	}
+
+	// fill back checksums
+	*checksums = *targetChecksums
+	return poolPath, true, nil
+}

azure/package_pool_test.go

@@ -0,0 +1,257 @@
+package azure
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type PackagePoolSuite struct {
+	accountName, accountKey, endpoint string
+	pool, prefixedPool                *PackagePool
+	debFile                           string
+	cs                                aptly.ChecksumStorage
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
+	c.Assert(err, IsNil)
+
+	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = files.NewMockChecksumStorage()
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
+	})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	var checksum utils.ChecksumInfo
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	// import as different name
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	// double import, should be ok
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with path-relevant checksums already filled in
+	checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
+	path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	checksum := utils.ChecksumInfo{}
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with checksums missing (that aren't needed to find the path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
+		SHA256: "abc",
+		Size:   checksum.Size,
+	}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksums (that aren't needed to find the path)
+	// and no info in checksum storage
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	checksum = utils.ChecksumInfo{Size: 13455}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestSize(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Size("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}

azure/public.go

@@ -0,0 +1,287 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Azure)
+type PublishedStorage struct {
+	prefix    string
+	az        *azContext
+	pathCache map[string]map[string]string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorage creates published storage from Azure storage credentials
+func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PublishedStorage{az: azctx}, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return storage.az.String()
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(_ string) error {
+	// no op for Azure
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+
+	sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename)
+	if err != nil {
+		return err
+	}
+
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.az.putFile(path, source, sourceMD5)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
+	path = storage.az.blobPath(path)
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range filelist {
+		blob := filepath.Join(path, filename)
+		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
+		if err != nil {
+			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
+		}
+	}
+
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	path = storage.az.blobPath(path)
+	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
+	}
+	return err
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
+
+	relFilePath := filepath.Join(publishedRelPath, fileName)
+	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	poolPath := storage.az.blobPath(prefixRelFilePath)
+
+	if storage.pathCache == nil {
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
+		}
+
+		pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			pathCache[paths[i]] = md5s[i]
+		}
+		storage.pathCache[publishedPrefix] = pathCache
+	}
+
+	destinationMD5, exists := pathCache[relFilePath]
+	sourceMD5 := sourceChecksums.MD5
+
+	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+		}
+	}
+
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.az.putFile(relFilePath, source, sourceMD5)
+	if err == nil {
+		pathCache[relFilePath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
+	}
+
+	return err
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	paths, _, err := storage.az.internalFilelist(prefix, nil)
+	return paths, err
+}
+
+// Internal copy or move implementation
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
+	const leaseDuration = 30
+	leaseID := uuid.NewString()
+
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	srcBlobClient := containerClient.NewBlobClient(src)
+	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	}
+
+	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	}
+	defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
+
+	dstBlobClient := containerClient.NewBlobClient(dst)
+	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
+		Metadata: metadata,
+	})
+
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	copyStatus := *copyResp.CopyStatus
+	for {
+		if copyStatus == blob.CopyStatusTypeSuccess {
+			if move {
+				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
+					AccessConditions: &blob.AccessConditions{
+						LeaseAccessConditions: &blob.LeaseAccessConditions{
+							LeaseID: &leaseID,
+						},
+					},
+				})
+				return err
+			}
+			return nil
+		} else if copyStatus == blob.CopyStatusTypePending {
+			time.Sleep(1 * time.Second)
+			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
+			if err != nil {
+				return fmt.Errorf("error getting copy progress %s", dst)
+			}
+			copyStatus = *getMetadata.CopyStatus
+
+			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
+			if err != nil {
+				return fmt.Errorf("error renewing source blob lease %s", src)
+			}
+		} else {
+			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
+		}
+	}
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */)
+}
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	metadata := make(map[string]*string)
+	metadata["SymLink"] = &src
+	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	_, err := blobClient.GetProperties(context.Background(), nil)
+	if err != nil {
+		if isBlobNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
+	}
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	props, err := blobClient.GetProperties(context.Background(), nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to get blob properties: %v", err)
+	}
+
+	metadata := props.Metadata
+	if originalBlob, exists := metadata["original_blob"]; exists {
+		return *originalBlob, nil
+	}
+	return "", fmt.Errorf("error reading link %s: %v", path, err)
+}

azure/public_test.go

@@ -0,0 +1,374 @@
+package azure
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+        "bytes"
+
+        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type PublishedStorageSuite struct {
+	accountName, accountKey, endpoint string
+	storage, prefixedStorage          *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+const testContainerPrefix = "aptlytest-"
+
+func randContainer() string {
+	return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+	if n <= 0 {
+		panic("negative number")
+	}
+	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+	var bytes = make([]byte, n)
+	rand.Read(bytes)
+	for i, b := range bytes {
+		bytes[i] = alphanum[b%byte(len(alphanum))]
+	}
+	return string(bytes)
+}
+
+func (s *PublishedStorageSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	c.Assert(err, IsNil)
+	data, err := ioutil.ReadAll(resp.Body)
+	c.Assert(err, IsNil)
+	return data
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+        serviceClient := s.storage.az.client.ServiceClient()
+        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
+        blobClient := containerClient.NewBlobClient(path)
+        _, err := blobClient.GetProperties(context.Background(), nil)
+	c.Assert(err, NotNil)
+
+        storageError, ok := err.(*azcore.ResponseError)
+	c.Assert(ok, Equals, true)
+	c.Assert(storageError.StatusCode, Equals, 404)
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	hash := md5.Sum(data)
+        uploadOptions := &azblob.UploadStreamOptions{
+            HTTPHeaders: &blob.HTTPHeaders{
+            	BlobContentMD5: hash[:],
+            },
+        }
+        reader := bytes.NewReader(data)
+        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+
+	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b+c.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+	s.AssertNoFile(c, "a/b c.txt")
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestFilelistPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a 1", "a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a b", "a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	err = s.prefixedStorage.Remove("xyz")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlus(c *C) {
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.RenameFile("source.txt", "dest.txt")
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!"))
+
+	exists, err := s.storage.FileExists("source.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
+
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
+
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
+	c.Assert(err, IsNil)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
+	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in azure test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

cmd/api_serve.go

@@ -1,11 +1,21 @@
 package cmd
 
 import (
+	stdcontext "context"
+	"errors"
 	"fmt"
-	"github.com/smira/aptly/api"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/aptly-dev/aptly/api"
+	"github.com/aptly-dev/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"net/http"
 )
 
 func aptlyAPIServe(cmd *commander.Command, args []string) error {
@@ -18,16 +28,73 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	listen := context.Flags().Lookup("listen").Value.String()
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
+	if err != nil {
+		return err
+	}
 
+	// Try to recycle systemd fds for listening
+	listeners, err := activation.Listeners(true)
+	if len(listeners) > 1 {
+		panic("Got more than 1 listener from systemd. This is currently not supported!")
+	}
+	if err == nil && len(listeners) == 1 {
+		listener := listeners[0]
+		defer listener.Close()
+		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
+		err = http.Serve(listener, api.Router(context))
+		if err != nil {
+			return fmt.Errorf("unable to serve: %s", err)
+		}
+		return nil
+	}
+
+	// If there are none: use the listen argument.
+	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
-	err = http.ListenAndServe(listen, api.Router(context))
-	if err != nil {
+	server := http.Server{Handler: api.Router(context)}
+
+	sigchan := make(chan os.Signal, 1)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go (func() {
+		if _, ok := <-sigchan; ok {
+			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
+			context.TaskList().Wait()
+			server.Shutdown(stdcontext.Background())
+		}
+	})()
+	defer close(sigchan)
+
+	listenURL, err := url.Parse(listen)
+	if err == nil && listenURL.Scheme == "unix" {
+		file := listenURL.Path
+		os.Remove(file)
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
+		if err != nil {
+			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
+		}
+		defer listener.Close()
+
+		err = server.Serve(listener)
+	} else {
+		server.Addr = listen
+		err = server.ListenAndServe()
+	}
+
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("unable to serve: %s", err)
 	}
 
-	return err
+	return nil
 }
 
 func makeCmdAPIServe() *commander.Command {
@@ -36,16 +103,21 @@ func makeCmdAPIServe() *commander.Command {
 		UsageLine: "serve",
 		Short:     "start API HTTP service",
 		Long: `
-Stat HTTP server with aptly REST API.
+Start HTTP server with aptly REST API. The server can listen to either a port
+or Unix domain socket. When using a socket, Aptly will fully manage the socket
+file. This command also supports taking over from a systemd file descriptors to
+enable systemd socket activation.
 
 Example:
 
   $ aptly api serve -listen=:8080
+  $ aptly api serve -listen=unix:///tmp/aptly.sock
 `,
 		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
 	}
 
-	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening or unix://path to listen on a Unix domain socket")
+	cmd.Flag.Bool("no-lock", false, "don't lock the database")
 
 	return cmd
 

cmd/cmd.go

@@ -2,36 +2,67 @@
 package cmd
 
 import (
+	"bytes"
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"os"
+	"text/template"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"time"
+)
+
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	err = reflist.ForEach(func(key []byte) error {
-		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err2 != nil {
-			return err2
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	return
+	return PrintPackageList(list, "", "  ")
+
+}
+
+// PrintPackageList shows package list with specified format or default representation
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
+	if format == "" {
+		return result.ForEachIndexed(func(p *deb.Package) error {
+			context.Progress().Printf(prefix+"%s\n", p)
+			return nil
+		})
+	}
+
+	formatTemplate, err := template.New("format").Parse(format)
+	if err != nil {
+		return fmt.Errorf("error parsing -format template: %s", err)
+	}
+
+	return result.ForEachIndexed(func(p *deb.Package) error {
+		b := &bytes.Buffer{}
+		err = formatTemplate.Execute(b, p.ExtendedStanza())
+		if err != nil {
+			return fmt.Errorf("error applying template: %s", err)
+		}
+		context.Progress().Printf(prefix+"%s\n", b.String())
+		return nil
+	})
+
 }
 
 // LookupOption checks boolean flag with default (usually config) and command-line
@@ -80,12 +111,15 @@ package environment to new version.`,
 		},
 	}
 
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/config_show.go

@@ -3,20 +3,32 @@ package cmd
 import (
 	"encoding/json"
 	"fmt"
+
 	"github.com/smira/commander"
+	"gopkg.in/yaml.v3"
 )
 
-func aptlyConfigShow(cmd *commander.Command, args []string) error {
+func aptlyConfigShow(_ *commander.Command, _ []string) error {
+	showYaml := context.Flags().Lookup("yaml").Value.Get().(bool)
 
 	config := context.Config()
-	prettyJSON, err := json.MarshalIndent(config, "", "    ")
 
-	if err != nil {
-		return fmt.Errorf("unable to dump the config file: %s", err)
+	if showYaml {
+		yamlData, err := yaml.Marshal(&config)
+		if err != nil {
+			return fmt.Errorf("error marshaling to YAML: %s", err)
+		}
+
+		fmt.Println(string(yamlData))
+	} else {
+		prettyJSON, err := json.MarshalIndent(config, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to dump the config file: %s", err)
+		}
+
+		fmt.Println(string(prettyJSON))
 	}
 
-	fmt.Println(string(prettyJSON))
-
 	return nil
 }
 
@@ -34,5 +46,6 @@ Example:
 
 `,
 	}
+	cmd.Flag.Bool("yaml", false, "show yaml config")
 	return cmd
 }

cmd/context.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	ctx "github.com/smira/aptly/context"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/flag"
 )
 
@@ -29,3 +29,8 @@ func InitContext(flags *flag.FlagSet) error {
 
 	return err
 }
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/db_cleanup.go

@@ -2,10 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
 	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
 )
 
 // aptly db cleanup
@@ -17,61 +20,137 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
+	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
 
-	context.Progress().Printf("Loading mirrors, local repos, snapshots and published repos...\n")
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+	// used only in verbose mode to report package use source
+	packageRefSources := map[string][]string{}
+
+	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
+	}
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
+		}
+
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
+	}
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
+
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+
+			if verbose {
+				description := fmt.Sprintf("local repo %s", repo.Name)
+				repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
+
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
+	}
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
+
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
+		}
+
 		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+		if verbose {
+			description := fmt.Sprintf("snapshot %s", snapshot.Name)
+			snapshot.RefList().ForEach(func(key []byte) error {
+				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				return nil
+			})
+		}
 		return nil
 	})
 	if err != nil {
 		return err
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
-		if published.SourceKind != "local" {
+	collectionFactory.Flush()
+
+	if verbose {
+		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
+	}
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+		if verbose {
+			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
+		}
+		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
-		if err != nil {
-			return err
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+		if e != nil {
+			return e
 		}
 
 		for _, component := range published.Components() {
 			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			if verbose {
+				description := fmt.Sprintf("published repository %s:%s/%s component %s",
+					published.Storage, published.Prefix, published.Distribution, component)
+				published.RefList(component).ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
 		}
 		return nil
 	})
@@ -79,39 +158,70 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	// ... and compare it to the list of all packages
-	context.Progress().Printf("Loading list of all packages...\n")
-	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
+	collectionFactory.Flush()
 
-	toDelete := allPackageRefs.Substract(existingPackageRefs)
+	// ... and compare it to the list of all packages
+	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
+	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
 
 	// database can't err as collection factory already constructed
 	db, _ := context.Database()
-	db.StartBatch()
-	err = toDelete.ForEach(func(ref []byte) error {
-		return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
-	})
-	if err != nil {
-		return err
+
+	if toDelete.Len() > 0 {
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of package keys to delete:@|")
+			err = toDelete.ForEach(func(ref []byte) error {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", string(ref))
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+		}
+
+		if !dryRun {
+			batch := db.CreateBatch()
+			err = toDelete.ForEach(func(ref []byte) error {
+				return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+			})
+			if err != nil {
+				return fmt.Errorf("unable to delete by key: %s", err)
+			}
+
+			err = batch.Write()
+			if err != nil {
+				return fmt.Errorf("unable to write to DB: %s", err)
+			}
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	err = db.FinishBatch()
-	if err != nil {
-		return fmt.Errorf("unable to write to DB: %s", err)
-	}
+	collectionFactory.Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)
-	context.Progress().Printf("Building list of files referenced by packages...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
 		if err2 != nil {
-			return err2
+			tail := ""
+			if verbose {
+				tail = fmt.Sprintf(" (sources: %s)", strings.Join(packageRefSources[string(key)], ", "))
+			}
+			if dryRun {
+				context.Progress().ColoredPrintf("@{r!}Unresolvable package reference, skipping (-dry-run): %s: %s%s",
+					string(key), err2, tail)
+				return nil
+			}
+			return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
 		}
 		paths, err2 := pkg.FilepathList(context.PackagePool())
 		if err2 != nil {
@@ -130,7 +240,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	context.Progress().ShutdownBar()
 
 	// build a list of files in the package pool
-	context.Progress().Printf("Building list of files in package pool...\n")
+	context.Progress().ColoredPrintf("@{w!}Building list of files in package pool...@|")
 	existingFiles, err := context.PackagePool().FilepathList(context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to collect file paths: %s", err)
@@ -140,28 +250,43 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
 
 	// delete files that are no longer referenced
-	context.Progress().Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced files (%d)...@|", len(filesToDelete))
 
 	if len(filesToDelete) > 0 {
-		context.Progress().InitBar(int64(len(filesToDelete)), false)
-
-		var size, totalSize int64
-		for _, file := range filesToDelete {
-			size, err = context.PackagePool().Remove(file)
-			if err != nil {
-				return err
+		if verbose {
+			context.Progress().ColoredPrintf("@{r}List of files to be deleted:@|")
+			for _, file := range filesToDelete {
+				context.Progress().ColoredPrintf(" - @{r}%s@|", file)
 			}
-
-			context.Progress().AddBar(1)
-			totalSize += size
 		}
-		context.Progress().ShutdownBar()
 
-		context.Progress().Printf("Disk space freed: %s...\n", utils.HumanBytes(totalSize))
+		if !dryRun {
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
+
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return err
+				}
+
+				context.Progress().AddBar(1)
+				totalSize += size
+			}
+			context.Progress().ShutdownBar()
+
+			context.Progress().ColoredPrintf("@{w!}Disk space freed: %s...@|", utils.HumanBytes(totalSize))
+		} else {
+			context.Progress().ColoredPrintf("@{y!}Skipped file deletion, as -dry-run has been requested.@|")
+		}
 	}
 
-	context.Progress().Printf("Compacting database...\n")
-	err = db.CompactDB()
+	if !dryRun {
+		context.Progress().ColoredPrintf("@{w!}Compacting database...@|")
+		err = db.CompactDB()
+	} else {
+		context.Progress().ColoredPrintf("@{y!}Skipped DB compaction, as -dry-run has been requested.@|")
+	}
 
 	return err
 }
@@ -181,5 +306,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("verbose", false, "be verbose when loading objects/removing them")
+	cmd.Flag.Bool("dry-run", false, "don't delete anything")
+
 	return cmd
 }

cmd/db_recover.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/database"
 	"github.com/smira/commander"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
@@ -15,7 +16,7 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = database.RecoverDB(context.DBPath())
+	err = goleveldb.RecoverDB(context.DBPath())
 
 	return err
 }

cmd/graph.go

@@ -3,12 +3,16 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
 )
 
 func aptlyGraph(cmd *commander.Command, args []string) error {
@@ -19,24 +23,34 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	layout := context.Flags().Lookup("layout").Value.String()
+
 	fmt.Printf("Generating graph...\n")
-	graph, err := deb.BuildGraph(context.CollectionFactory())
+	collectionFactory := context.NewCollectionFactory()
+	graph, err := deb.BuildGraph(collectionFactory, layout)
 	if err != nil {
 		return err
 	}
 
 	buf := bytes.NewBufferString(graph.String())
 
-	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	tempfile, err := os.CreateTemp("", "aptly-graph")
 	if err != nil {
 		return err
 	}
 	tempfile.Close()
 	os.Remove(tempfile.Name())
 
-	tempfilename := tempfile.Name() + ".png"
+	format := context.Flags().Lookup("format").Value.String()
+	output := context.Flags().Lookup("output").Value.String()
 
-	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	if filepath.Ext(output) != "" {
+		format = filepath.Ext(output)[1:]
+	}
+
+	tempfilename := tempfile.Name() + "." + format
+
+	command := exec.Command("dot", "-T"+format, "-o"+tempfilename)
 	command.Stderr = os.Stderr
 
 	stdin, err := command.StdinPipe()
@@ -64,15 +78,40 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	err = exec.Command("open", tempfilename).Run()
-	if err != nil {
-		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
-		err = nil
+	if output != "" {
+		err = utils.CopyFile(tempfilename, output)
+		if err != nil {
+			return fmt.Errorf("unable to copy %s -> %s: %s", tempfilename, output, err)
+		}
+
+		fmt.Printf("Output saved to %s\n", output)
+		_ = os.Remove(tempfilename)
+	} else {
+		command := getOpenCommand()
+		fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename)
+
+		args := strings.Split(command, " ")
+
+		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
+		viewer.Stderr = os.Stderr
+		err = viewer.Start()
 	}
 
 	return err
 }
 
+// getOpenCommand tries to guess command to open image for OS
+func getOpenCommand() string {
+	switch runtime.GOOS {
+	case "darwin":
+		return "/usr/bin/open"
+	case "windows":
+		return "cmd /c start"
+	default:
+		return "xdg-open"
+	}
+}
+
 func makeCmdGraph() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyGraph,
@@ -89,5 +128,9 @@ Example:
 `,
 	}
 
+	cmd.Flag.String("format", "png", "render graph to specified format (png, svg, pdf, etc.)")
+	cmd.Flag.String("output", "", "specify output filename, default is to open result in viewer")
+	cmd.Flag.String("layout", "horizontal", "create a more 'vertical' or a more 'horizontal' graph layout")
+
 	return cmd
 }

cmd/mirror.go

@@ -1,25 +1,26 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
-func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
-		return nil, nil
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
+	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
 	}
 
-	keyRings := flags.Lookup("keyring").Value.Get().([]string)
-
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
@@ -18,6 +19,11 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -35,14 +41,15 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs)
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
+	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
 
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
@@ -56,12 +63,13 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -92,11 +100,14 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
+	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -14,8 +15,9 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -27,7 +29,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -39,7 +41,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
+	err = collectionFactory.RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/mirror_edit.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/query"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -14,7 +16,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -24,16 +27,25 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter = flag.Value.String()
+			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
+		case "ignore-signatures":
+			ignoreSignatures = true
 		}
 	})
 
@@ -50,14 +62,23 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
 
-		err = repo.Fetch(context.Downloader(), nil)
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -82,10 +103,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_list.go

@@ -1,24 +1,38 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	if jsonFlag {
+		return aptlyMirrorListJSON(cmd, args)
+	}
+
+	return aptlyMirrorListTxt(cmd, args)
+}
+
+func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -51,6 +65,32 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdMirrorList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorList,
@@ -65,6 +105,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/mirror_rename.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,7 +20,8 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
@@ -29,13 +31,13 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	_, err = collectionFactory.RemoteRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/mirror_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdMirrorSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search mirror for packages matching query",
 		Long: `
 Command search displays list of packages in mirror that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
@@ -21,6 +23,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -1,29 +1,44 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"sort"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorShowJSON(cmd, args)
+	}
+
+	return aptlyMirrorShowTxt(cmd, args)
+}
+
+func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -36,21 +51,21 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	downloadSources := "no"
+	downloadSources := No
 	if repo.DownloadSources {
-		downloadSources = "yes"
+		downloadSources = Yes
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
-	downloadUdebs := "no"
+	downloadUdebs := No
 	if repo.DownloadUdebs {
-		downloadUdebs = "yes"
+		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
-		filterWithDeps := "no"
+		filterWithDeps := No
 		if repo.FilterWithDeps {
-			filterWithDeps = "yes"
+			filterWithDeps = Yes
 		}
 		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
 	}
@@ -71,13 +86,56 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList())
+			ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
 	return err
 }
 
+func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				repo.Packages = append(repo.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(repo.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdMirrorShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorShow,
@@ -93,6 +151,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
 
 	return cmd

cmd/mirror_update.go

@@ -2,14 +2,16 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/utils"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
-	"os/signal"
-	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
@@ -21,12 +23,13 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -39,20 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -67,7 +74,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 
 		var oldLen, newLen int
-		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -79,23 +86,27 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		queue        []deb.PackageDownloadTask
 	)
 
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	defer func() {
 		// on any interruption, unlock the mirror
-		err := context.ReOpenDatabase()
+		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+			collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
 	repo.MarkAsUpdating()
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -105,64 +116,158 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	// Catch ^C
-	sigch := make(chan os.Signal)
-	signal.Notify(sigch, os.Interrupt)
+	context.GoContextHandleSignals()
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
 
 	// Download from the queue
-	context.Progress().InitBar(downloadSize, true)
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
 
-	// Download all package files
-	ch := make(chan error, count)
+	downloadQueue := make(chan int)
+
+	var (
+		errors  []string
+		errLock sync.Mutex
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
+	}
 
-	// In separate goroutine (to avoid blocking main), push queue to downloader
 	go func() {
-		for _, task := range queue {
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+		for idx := range queue {
+			select {
+			case downloadQueue <- idx:
+			case <-context.Done():
+				return
+			}
 		}
-
-		// We don't need queue after this point
-		queue = nil
+		close(downloadQueue)
 	}()
 
-	// Wait for all downloads to finish
-	errors := make([]string, 0)
+	var wg sync.WaitGroup
 
-	for count > 0 {
-		select {
-		case <-sigch:
-			signal.Stop(sigch)
-			return fmt.Errorf("unable to update: interrupted")
-		case err = <-ch:
-			if err != nil {
-				errors = append(errors, err.Error())
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
+				select {
+				case idx, ok := <-downloadQueue:
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+						task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+					} else {
+						var file *os.File
+						file, e = os.CreateTemp("", task.File.Filename)
+						if e == nil {
+							task.TempDownPath = file.Name()
+							file.Close()
+						}
+					}
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						context,
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreChecksums)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					task.Done = true
+				case <-context.Done():
+					return
+				}
 			}
-			count--
-		}
+		}()
 	}
 
+	// Wait for all download goroutines to finish
+	wg.Wait()
+
 	context.Progress().ShutdownBar()
-	signal.Stop(sigch)
-
-	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
-	}
 
 	err = context.ReOpenDatabase()
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	repo.FinalizeDownload()
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	defer func() {
+		for _, task := range queue {
+			if task.TempDownPath == "" {
+				continue
+			}
+
+			if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+				fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+			}
+		}
+	}()
+
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
+
+	for idx := range queue {
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(collectionFactory, context.Progress())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
 	return err
 }
 
@@ -186,7 +291,10 @@ Example:
 	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/package_search.go

@@ -2,33 +2,45 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyPackageSearch(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 1 {
+	var (
+		err error
+		q   deb.PackageQuery
+	)
+
+	if len(args) > 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
-	if err != nil {
-		return fmt.Errorf("unable to search: %s", err)
+	if len(args) == 1 {
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+	} else {
+		q = &deb.MatchAllQuery{}
 	}
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 	if result.Len() == 0 {
 		return fmt.Errorf("no results")
 	}
 
-	result.ForEach(func(p *deb.Package) error {
-		context.Progress().Printf("%s\n", p)
-		return nil
-	})
+	format := context.Flags().Lookup("format").Value.String()
+	PrintPackageList(result, format, "")
 
 	return err
 }
@@ -36,10 +48,13 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 func makeCmdPackageSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPackageSearch,
-		UsageLine: "search <package-query>",
+		UsageLine: "search [<package-query>]",
 		Short:     "search for packages matching query",
 		Long: `
-Command search displays list of packages in whole DB that match package query
+Command search displays list of packages in whole DB that match package query.
+
+Use '@file' to read query from file or '@-' for stdin.
+If query is not specified, all the packages are displayed.
 
 Example:
 
@@ -48,5 +63,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("format", "", "custom format for result printing")
+
 	return cmd
 }

cmd/package_show.go

@@ -3,18 +3,20 @@ package cmd
 import (
 	"bufio"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
-func printReferencesTo(p *deb.Package) (err error) {
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -27,10 +29,10 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return err
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -43,21 +45,18 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
-			return err
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if e != nil {
+			return e
 		}
 		if snapshot.RefList().Has(p) {
 			fmt.Printf("  snapshot %s\n", snapshot)
 		}
 		return nil
 	})
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }
 
 func aptlyPackageShow(cmd *commander.Command, args []string) error {
@@ -67,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -77,20 +80,28 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 	w := bufio.NewWriter(os.Stdout)
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false)
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
 		w.Flush()
 		fmt.Printf("\n")
 
 		if withFiles {
 			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
 			for _, f := range p.Files() {
-				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				var path string
+				path, err = f.GetPoolPath(packagePool)
 				if err != nil {
 					return err
 				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
 				fmt.Printf("  %s\n", path)
 			}
 			fmt.Printf("\n")
@@ -98,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p)
+			printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 
@@ -123,9 +134,11 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
-    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+    $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'
 `,
 		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
 	}

cmd/publish.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
@@ -34,9 +34,26 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishDrop(),
 			makeCmdPublishList(),
 			makeCmdPublishRepo(),
+			makeCmdPublishShow(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishSource(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
 		},
 	}
 }
+
+func makeCmdPublishSource() *commander.Command {
+	return &commander.Command{
+		UsageLine: "source",
+		Short:     "manage sources of published repository",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSourceAdd(),
+			makeCmdPublishSourceDrop(),
+			makeCmdPublishSourceList(),
+			makeCmdPublishSourceRemove(),
+			makeCmdPublishSourceReplace(),
+			makeCmdPublishSourceUpdate(),
+		},
+	}
+}

cmd/publish_drop.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -22,8 +23,11 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 
 	storage, prefix := deb.ParsePrefix(param)
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress())
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		collectionFactory, context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -48,5 +52,8 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+
 	return cmd
 }

cmd/publish_list.go

@@ -1,27 +1,44 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
+	"os"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishListJSON(cmd, args)
+	}
+
+	return aptlyPublishListTxt(cmd, args)
+}
+
+func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
 		}
 
 		if raw {
@@ -60,6 +77,42 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		repos = append(repos, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GetPath() < repos[j].GetPath()
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdPublishList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishList,
@@ -74,6 +127,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/publish_repo.go

@@ -37,13 +37,21 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_show.go

@@ -0,0 +1,127 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+)
+
+func aptlyPublishShow(cmd *commander.Command, args []string) error {
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishShowJSON(cmd, args)
+	}
+
+	return aptlyPublishShowTxt(cmd, args)
+}
+
+func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	if repo.Storage != "" {
+		fmt.Printf("Storage: %s\n", repo.Storage)
+	}
+	fmt.Printf("Prefix: %s\n", repo.Prefix)
+	if repo.Distribution != "" {
+		fmt.Printf("Distribution: %s\n", repo.Distribution)
+	}
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
+
+	fmt.Printf("Sources:\n")
+	for _, component := range repo.Components() {
+		sourceID := repo.Sources[component]
+		var name string
+		if repo.SourceKind == deb.SourceSnapshot {
+			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		} else if repo.SourceKind == deb.SourceLocalRepo {
+			source, e := collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+			if e != nil {
+				continue
+			}
+			name = source.Name
+		}
+
+		if name != "" {
+			fmt.Printf("  %s: %s [%s]\n", component, name, repo.SourceKind)
+		}
+	}
+
+	return err
+}
+
+func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
+func makeCmdPublishShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishShow,
+		UsageLine: "show <distribution> [[<endpoint>:]<prefix>]",
+		Short:     "shows details of published repository",
+		Long: `
+Command show displays full information of a published repository.
+
+Example:
+
+    $ aptly publish show wheezy
+`,
+	}
+
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -2,18 +2,20 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	collectionFactory := context.NewCollectionFactory()
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -34,7 +36,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		message string
 	)
 
-	if cmd.Name() == "snapshot" {
+	if cmd.Name() == "snapshot" { // nolint: goconst
 		var (
 			snapshot     *deb.Snapshot
 			emptyWarning = false
@@ -42,12 +44,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -70,7 +72,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		if emptyWarning {
 			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
 		}
-	} else if cmd.Name() == "repo" {
+	} else if cmd.Name() == "repo" { // nolint: goconst
 		var (
 			localRepo    *deb.LocalRepo
 			emptyWarning = false
@@ -78,12 +80,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			localRepo, err = collectionFactory.LocalRepoCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -111,17 +113,50 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = cmd.Flag.Lookup("origin").Value.String()
-	published.Label = cmd.Flag.Lookup("label").Value.String()
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
+	published.Label = context.Flags().Lookup("label").Value.String()
+	published.Suite = context.Flags().Lookup("suite").Value.String()
+	published.Codename = context.Flags().Lookup("codename").Value.String()
 
-	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
+	published.SkipContents = context.Config().SkipContentsPublishing
+
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	published.SkipBz2 = context.Config().SkipBz2Publishing
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -132,16 +167,15 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
-			"the same package pool.\n")
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -156,14 +190,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
 		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
 			localStorage.PublicPath())
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
 	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
 		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
@@ -199,13 +233,21 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
-	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_source_add.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if exists {
+			return fmt.Errorf("unable to add: component '%s' has already been added", component)
+		}
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceAdd,
+		UsageLine: "add <distribution> <source>",
+		Short:     "add source components to a published repo",
+		Long: `
+The command adds components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source add -component=contrib wheezy ppa wheezy-contrib
+
+This command assigns the snapshot wheezy-contrib to the component contrib and
+adds it to published repository revision of ppa/wheezy.
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_drop.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published.DropRevision()
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("Source changes have been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishSourceDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceDrop,
+		UsageLine: "drop <distribution>",
+		Short:     "drop pending source component changes of a published repository",
+		Long: `
+Remove all pending changes what would be applied with a subsequent 'aptly publish update'.
+
+Example:
+
+    $ aptly publish source drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_list.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	if published.Revision == nil {
+		return fmt.Errorf("unable to list: no source changes exist")
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishSourceListJSON(published)
+	}
+
+	return aptlyPublishSourceListTxt(published)
+}
+
+func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	fmt.Printf("Sources:\n")
+	for _, component := range revision.Components() {
+		name := revision.Sources[component]
+		fmt.Printf("  %s: %s [%s]\n", component, name, published.SourceKind)
+	}
+
+	return nil
+}
+
+func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	output, err := json.MarshalIndent(revision.SourceList(), "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+func makeCmdPublishSourceList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceList,
+		UsageLine: "list <distribution>",
+		Short:     "lists revision of published repository",
+		Long: `
+Command lists sources of a published repository.
+
+Example:
+
+    $ aptly publish source list wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
+	}
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_remove.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(components) == 0 {
+		return fmt.Errorf("unable to remove: missing components, specify at least one component")
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for _, component := range components {
+		name, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to remove: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		delete(sources, component)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceRemove,
+		UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
+		Short:     "remove source components from a published repo",
+		Long: `
+The command removes source components (snapshot / local repo) from a published repository.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be removed, e.g.:
+
+Example:
+
+	$ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_replace.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+	context.Progress().Printf("Replacing source list...\n")
+	clear(sources)
+
+	for i, component := range components {
+		name := names[i]
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceReplace() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceReplace,
+		UsageLine: "replace <distribution> <source>",
+		Short:     "replace the source components of a published repository",
+		Long: `
+The command replaces the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_update.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceUpdate,
+		UsageLine: "update <distribution> <source>",
+		Short:     "update the source components of a published repository",
+		Long: `
+The command updates the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source update -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -2,15 +2,19 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"strings"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
-	var err error
+	var (
+		err   error
+		names []string
+	)
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
@@ -22,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	distribution := args[0]
 	param := "."
 
-	var (
-		names    []string
-		snapshot *deb.Snapshot
-	)
-
 	if len(args) == len(components)+2 {
 		param = args[1]
 		names = args[2:]
@@ -38,18 +37,19 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
-	if published.SourceKind != "snapshot" {
-		return fmt.Errorf("unable to update: not a snapshot publish")
+	if published.SourceKind != deb.SourceSnapshot {
+		return fmt.Errorf("unable to switch: not a published snapshot repository")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	publishedComponents := published.Components()
@@ -61,17 +61,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
 	}
 
+	snapshotCollection := collectionFactory.SnapshotCollection()
 	for i, component := range components {
 		if !utils.StrSliceHasItem(publishedComponents, component) {
-			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+			return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
 		}
 
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		snapshot, err := snapshotCollection.ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -90,23 +91,37 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
 	}
 
-	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -114,15 +129,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
-		Short:     "update published repository by switching to new snapshot",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
+		Short:     "update published repository by switching to new source",
 		Long: `
-Command switches in-place published snapshots with new snapshot contents. All
+Command switches in-place published snapshots with new source contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
 For multiple component repositories, flag -component should be given with
-list of components to update. Corresponding snapshots should be given in the
+list of components to update. Corresponding sources should be given in the
 same order, e.g.:
 
 	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
@@ -139,12 +154,16 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -24,23 +25,20 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "local" {
-		return fmt.Errorf("unable to update: not a local repository publish")
-	}
-
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	components := published.Components()
-	for _, component := range components {
-		published.UpdateLocalRepo(component)
+	result, err := published.Update(collectionFactory, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -54,23 +52,39 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-contents") {
+		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
-	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -79,15 +93,21 @@ func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
 		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
-		Short:     "update published local repository",
+		Short:     "update published repository",
 		Long: `
-Command re-publishes (updates) published local repository. <distribution>
-and <prefix> should be occupied with local repository published
-using command aptly publish repo. Update happens in-place with
-minimum possible downtime for published repository.
+The command updates updates a published repository after applying pending changes to the sources.
 
-For multiple component published repositories, all local repositories
-are updated.
+For published local repositories:
+
+    * update to match local repository contents
+
+For published snapshots:
+
+    * switch components to new snapshot
+
+The update happens in-place with minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories are updated.
 
 Example:
 
@@ -98,11 +118,15 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/repo.go

@@ -21,6 +21,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
 			makeCmdRepoSearch(),
+			makeCmdRepoInclude(),
 		},
 	}
 }

cmd/repo_add.go

@@ -2,12 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"os"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"os"
 )
 
 func aptlyRepoAdd(cmd *commander.Command, args []string) error {
@@ -19,46 +20,47 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	var packageFiles, failedFiles []string
+	var packageFiles, otherFiles, failedFiles []string
 
-	packageFiles, failedFiles, err = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
-	if err != nil {
-		return fmt.Errorf("unable to collect package files: %s", err)
-	}
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()})
+		collectionFactory.PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		collectionFactory.ChecksumCollection)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
@@ -67,7 +69,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}
@@ -89,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 func makeCmdRepoAdd() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoAdd,
-		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		UsageLine: "add <name> (<package file.deb>|<directory>)...",
 		Short:     "add packages to local repository",
 		Long: `
 Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.

cmd/repo_create.go

@@ -2,14 +2,15 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if len(args) != 1 {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
@@ -18,7 +19,32 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
 	repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
 
-	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		repo.Uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 {
+		var snapshot *deb.Snapshot
+
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(args[3])
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to load source snapshot: %s", err)
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}
@@ -30,16 +56,21 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 func makeCmdRepoCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoCreate,
-		UsageLine: "create <name>",
+		UsageLine: "create <name> [ from snapshot <snapshot> ]",
 		Short:     "create local repository",
 		Long: `
 Create local package repository. Repository would be empty when
 created, packages could be added from files, copied or moved from
 another local repository or imported from the mirror.
 
+If local package repository is created from snapshot, repo initial
+contents are copied from snapsot contents.
+
 Example:
 
   $ aptly repo create testing
+
+  $ aptly repo create mysql35 from snapshot mysql-35-2017
 `,
 		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
 	}
@@ -47,6 +78,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "main", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -14,17 +15,18 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	published := collectionFactory.PublishedRepoCollection().ByLocalRepo(repo)
 	if len(published) > 0 {
 		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -36,7 +38,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -48,7 +50,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
+	err = collectionFactory.LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/repo_edit.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"fmt"
+
+	"github.com/AlekSi/pointer"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -13,29 +16,44 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	if context.Flags().Lookup("comment").Value.String() != "" {
-		repo.Comment = context.Flags().Lookup("comment").Value.String()
+	var uploadersFile *string
+
+	context.Flags().Visit(func(flag *flag.Flag) {
+		switch flag.Name {
+		case "comment":
+			repo.Comment = flag.Value.String()
+		case "distribution":
+			repo.DefaultDistribution = flag.Value.String()
+		case "component":
+			repo.DefaultComponent = flag.Value.String()
+		case "uploaders-file":
+			uploadersFile = pointer.ToString(flag.Value.String())
+		}
+	})
+
+	if uploadersFile != nil {
+		if *uploadersFile != "" {
+			repo.Uploaders, err = deb.NewUploadersFromFile(*uploadersFile)
+			if err != nil {
+				return err
+			}
+		} else {
+			repo.Uploaders = nil
+		}
 	}
 
-	if context.Flags().Lookup("distribution").Value.String() != "" {
-		repo.DefaultDistribution = context.Flags().Lookup("distribution").Value.String()
-	}
-
-	if context.Flags().Lookup("component").Value.String() != "" {
-		repo.DefaultComponent = context.Flags().Lookup("component").Value.String()
-	}
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -63,6 +81,7 @@ Example:
 	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
 	cmd.Flag.String("distribution", "", "default distribution when publishing")
 	cmd.Flag.String("component", "", "default component when publishing")
+	cmd.Flag.String("uploaders-file", "", "uploaders.json to be used when including .changes into this repository")
 
 	return cmd
 }

cmd/repo_include.go

@@ -0,0 +1,115 @@
+package cmd
+
+import (
+	"fmt"
+	"text/template"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyRepoInclude(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	verifier, err := getVerifier(context.Flags())
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	if verifier == nil {
+		verifier = context.GetVerifier()
+	}
+
+	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
+	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+	collectionFactory := context.NewCollectionFactory()
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
+
+	uploaders := (*deb.Uploaders)(nil)
+	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
+	if uploadersFile != "" {
+		uploaders, err = deb.NewUploadersFromFile(uploadersFile)
+		if err != nil {
+			return err
+		}
+
+		for i := range uploaders.Rules {
+			uploaders.Rules[i].CompiledCondition, err = query.Parse(uploaders.Rules[i].Condition)
+			if err != nil {
+				return fmt.Errorf("error parsing query %s: %s", uploaders.Rules[i].Condition, err)
+			}
+		}
+	}
+
+	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
+
+	var changesFiles, failedFiles, failedFiles2 []string
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
+		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+		context.PackagePool(), collectionFactory.ChecksumCollection,
+		uploaders, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if len(failedFiles) > 0 {
+		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
+		for _, file := range failedFiles {
+			context.Progress().ColoredPrintf("  %s", file)
+		}
+
+		return fmt.Errorf("some files failed to be added")
+	}
+
+	return err
+}
+
+func makeCmdRepoInclude() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoInclude,
+		UsageLine: "include (<file.changes>|<directory>)...",
+		Short:     "add packages to local repositories based on .changes files",
+		Long: `
+Command include looks for .changes files in list of arguments or specified directories. Each
+.changes file is verified, parsed, referenced files are put into separate temporary directory
+and added into local repository. Successfully imported files are removed by default.
+
+Additionally uploads could be restricted with 'uploaders.json' file. Rules in this file control
+uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
+
+Example:
+
+  $ aptly repo include -repo=foo-release incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-include", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("no-remove-files", false, "don't remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
+	cmd.Flag.String("repo", "{{.Distribution}}", "which repo should files go to, defaults to Distribution field of .changes file")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of .changes file signature")
+	cmd.Flag.Bool("accept-unsigned", false, "accept unsigned .changes files")
+	cmd.Flag.String("uploaders-file", "", "path to uploaders.json file")
+
+	return cmd
+}

cmd/repo_list.go

@@ -1,30 +1,44 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/commander"
 	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoListJSON(cmd, args)
+	}
+
+	return aptlyRepoListTxt(cmd, args)
+}
+
+func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-			if err != nil {
-				return err
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
@@ -57,6 +71,36 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdRepoList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoList,
@@ -71,6 +115,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/repo_move.go

@@ -2,11 +2,12 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
-	"sort"
 )
 
 func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
@@ -18,12 +19,13 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	command := cmd.Name()
 
-	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -33,8 +35,8 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		srcRepo    *deb.LocalRepo
 	)
 
-	if command == "copy" || command == "move" {
-		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	if command == "copy" || command == "move" { // nolint: goconst
+		srcRepo, err = collectionFactory.LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -43,21 +45,21 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
 		srcRefList = srcRepo.RefList()
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+		srcRemoteRepo, err = collectionFactory.RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -73,12 +75,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -108,24 +110,35 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 	}
 
-	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
+	toProcess, err := srcList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            dstList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var verb string
 
-	if command == "move" {
+	if command == "move" { // nolint: goconst
 		verb = "moved"
-	} else if command == "copy" {
+	} else if command == "copy" { // nolint: goconst
 		verb = "copied"
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		verb = "imported"
 	}
 
@@ -135,7 +148,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcList.Remove(p)
 		}
 		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
@@ -150,15 +163,15 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -177,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -2,8 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -17,33 +18,38 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
 	}
 
 	list.PrepareIndex()
-	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	toRemove, err := list.Filter(deb.FilterOptions{Queries: queries})
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -59,7 +65,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -79,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/repo_rename.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,18 +20,19 @@ func aptlyRepoRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.LocalRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	_, err = collectionFactory.LocalRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/repo_search.go

@@ -8,11 +8,13 @@ import (
 func makeCmdRepoSearch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotMirrorRepoSearch,
-		UsageLine: "search <name> <package-query>",
+		UsageLine: "search <name> [<package-query>]",
 		Short:     "search repo for packages matching query",
 		Long: `
 Command search displays list of packages in local repository that match package query
 
+If query is not specified, all the packages are displayed.
+
 Example:
 
     $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
@@ -21,6 +23,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+	cmd.Flag.String("format", "", "custom format for result printing")
 
 	return cmd
 }

cmd/repo_show.go

@@ -1,26 +1,42 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoShowJSON(cmd, args)
+	}
+
+	return aptlyRepoShowTxt(cmd, args)
+}
+
+func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -29,11 +45,56 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Comment: %s\n", repo.Comment)
 	fmt.Printf("Default Distribution: %s\n", repo.DefaultDistribution)
 	fmt.Printf("Default Component: %s\n", repo.DefaultComponent)
+	if repo.Uploaders != nil {
+		fmt.Printf("Uploaders: %s\n", repo.Uploaders)
+	}
 	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList())
+		ListPackagesRefList(repo.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	packageList := []string{}
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err == nil {
+				packageList = list.FullNames()
+			}
+		}
+
+		sort.Strings(packageList)
+	}
+
+	// merge the repo object with the package list
+	var output []byte
+	if output, err = json.MarshalIndent(struct {
+		*deb.LocalRepo
+		Packages []string
+	}{repo, packageList}, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -53,6 +114,7 @@ ex:
 		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/run.go

@@ -2,7 +2,9 @@ package cmd
 
 import (
 	"fmt"
-	ctx "github.com/smira/aptly/context"
+	"os"
+
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/commander"
 )
 
@@ -14,7 +16,7 @@ func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode
 			if !ok {
 				panic(r)
 			}
-			fmt.Println("ERROR:", fatal.Message)
+			fmt.Fprintln(os.Stderr, "ERROR:", fatal.Message)
 			returnCode = fatal.ReturnCode
 		}
 	}()

cmd/serve.go

@@ -2,16 +2,17 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
-	"github.com/smira/commander"
-	"github.com/smira/flag"
 	"net"
 	"net/http"
 	"os"
 	"sort"
 	"strings"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
 )
 
 func aptlyServe(cmd *commander.Command, args []string) error {
@@ -22,7 +23,19 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
+	// There are only two working options for aptly's rootDir:
+	//   1. rootDir does not exist, then we'll create it
+	//   2. rootDir exists and is writable
+	// anything else must fail.
+	// E.g.: Running the service under a different user may lead to a rootDir
+	// that exists but is not usable due to access permissions.
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
+	if err != nil {
+		return err
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	if collectionFactory.PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
@@ -44,13 +57,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
+	sources := make(sort.StringSlice, 0, collectionFactory.PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
-			return err
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		if e != nil {
+			return e
 		}
 
 		sources = append(sources, repo.String())
@@ -78,13 +91,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
 			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
-		if utils.StrSliceHasItem(repo.Architectures, "source") {
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
 				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot_create.go

@@ -2,7 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"github.com/smira/aptly/deb"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -12,13 +13,14 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+	collectionFactory := context.NewCollectionFactory()
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
 		repoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -28,7 +30,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -37,18 +39,18 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
-	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
 		// aptly snapshot create snap from repo repo
 		var repo *deb.LocalRepo
 
 		localRepoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -69,7 +71,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}
@@ -82,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotCreate,
-		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		UsageLine: "create <name> (from mirror <mirror-name> | from repo <repo-name> | empty)",
 		Short:     "creates snapshot of mirror (local repository) contents",
 		Long: `
 Command create <name> from mirror makes persistent immutable snapshot of remote

cmd/snapshot_diff.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -14,31 +15,32 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	}
 
 	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name-a> snapshot
-	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshotA, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	snapshotB, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_drop.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -14,18 +15,19 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
+	published := collectionFactory.PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -37,7 +39,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
+		snapshots := collectionFactory.SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -48,7 +50,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
+	err = collectionFactory.SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}