Merge tag 'upstream/1.6.0+ds1_alpha1' into debian/master

.dockerignore

@@ -0,0 +1,6 @@
+.go/
+.git/
+obj-x86_64-linux-gnu/
+unit.out
+aptly.test
+build/

.flake8

@@ -1,7 +1,7 @@
 [flake8]
-max-line-length = 200
+max-line-length = 240
 ignore = E126,E241,E741,W504
 include =
     system
 exclude =
-    system/env
+    system/env

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/workflows/ci.yml

@@ -1,5 +1,3 @@
-# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
-
 name: CI
 
 on:
@@ -13,32 +11,17 @@ on:
 defaults:
   run:
     # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
-    shell: bash --noprofile --norc -eo pipefail -x {0}
+    shell: bash --noprofile --norc -eo pipefail {0}
 
 env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
-  build:
+  test:
-    name: test
+    name: "Test (Ubuntu 22.04)"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
-    continue-on-error: ${{ matrix.allow_failure }}
+    continue-on-error: false
     timeout-minutes: 30
-    strategy:
-      fail-fast: true
-      matrix:
-        go: [1.16, 1.17, 1.18]
-        run_long_tests: [no]
-        allow_failure: [false]
-        include:
-          # Disable this for now as it's not clear how to select the latest master branch
-          # version of go using actions/setup-go@v2.
-          # - go: master
-          #   run_long_tests: no
-          #   allow_failure: true
-          - go: 1.17
-            run_long_tests: yes
-            allow_failure: false
 
     env:
       NO_FTP_ACCESS: yes
@@ -47,101 +30,253 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: Checkout repository
+      - name: "Install packages"
-        uses: actions/checkout@v2
-
-      - name: Setup Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go }}
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: v1.45.0
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install O/S packages
         run: |
           sudo apt-get update
-          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
 
-      - name: Install Python packages
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Run flake8"
         run: |
-          pip install six packaging appdirs virtualenv
+          make flake8
-          pip install -U pip setuptools
-          pip install -r system/requirements.txt
 
-      - name: Install Azurite
+      - name: "Read go version from go.mod"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Install Azurite"
         id: azuright
         uses: potatoqualitee/azuright@v1.1
         with:
           directory: ${{ runner.temp }}
 
-      - name: Get aptly version
+      - name: "Run Unit Tests"
-        run: |
-           make version
-
-      - name: Make
         env:
-          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
+          RUN_LONG_TESTS: 'yes'
-          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
           AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          make
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make test
 
-      - name: Upload code coverage
+      - name: "Run Benchmark"
-        if: matrix.run_long_tests
+        run: |
+          COVERAGE_DIR=${{ runner.temp }} make bench
+
+      - name: "Run System Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make system-test
+
+      - name: "Merge code coverage"
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+
+      - name: "Upload code coverage"
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.txt
 
-  release:
+  ci-debian-build:
-    name: release
+    name: "Build"
-    needs: build
+    needs: test
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        arch: ["amd64", "i386" , "arm64" , "armhf"]
+        include:
+          - name: "Debian 13/testing"
+            suite: trixie
+            image: debian:trixie-slim
+          - name: "Debian 12/bookworm"
+            suite: bookworm
+            image: debian:bookworm-slim
+          - name: "Debian 11/bullseye"
+            suite: bullseye
+            image: debian:bullseye-slim
+          - name: "Debian 10/buster"
+            suite: buster
+            image: debian:buster-slim
+          - name: "Ubuntu 24.04"
+            suite: noble
+            image: ubuntu:24.04
+          - name: "Ubuntu 22.04"
+            suite: jammy
+            image: ubuntu:22.04
+          - name: "Ubuntu 20.04"
+            suite: focal
+            image: ubuntu:20.04
+    container:
+      image: ${{ matrix.image }}
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: Checkout repository
+      - name: "Install packages"
-        uses: actions/checkout@v3
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf jq
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
         with:
-          # fetch the whole repot for `git describe` to
+          # fetch the whole repo for `git describe` to work
-          # work and get the nightly verion
           fetch-depth: 0
 
-      - name: Setup Go
+      - name: "Read go version from go.mod"
-        uses: actions/setup-go@v2
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
 
-      - name: Make Release
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Build Debian packages"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make dpkg DEBARCH=${{ matrix.arch }}
+
+      - name: "Check aptly credentials"
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          found=no
+          if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
+            found=yes
+          fi
+          echo "Aptly credentials available: $found"
+          echo "FOUND=$found" >> $GITHUB_OUTPUT
+        id: aptlycreds
+
+      - name: "Publish CI release to aptly"
+        if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
+
+      - name: "Publish release to aptly"
+        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
+
+  ci-binary-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, freebsd, darwin]
+        goarch: ["386", "amd64", "arm", "arm64"]
+        exclude:
+          - goos: darwin
+            goarch: 386
+          - goos: darwin
+            goarch: arm
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
         env:
           GOBIN: /usr/local/bin
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
         run: |
-          make release
+          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
 
-      - name: Publish nightly release to aptly
+      - name: "Upload Artifacts"
-        if: github.ref == 'refs/heads/master'
+        uses: actions/upload-artifact@v4
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh nightly
-
-      - name: Publish release to aptly
         if: startsWith(github.event.ref, 'refs/tags')
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh release
-
-      - name: Upload artifacts to GitHub Release
-        if: startsWith(github.event.ref, 'refs/tags')
-        uses: softprops/action-gh-release@v1
         with:
-          body: Release ${{ github.ref }} generated by the CI.
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
-          files: build/*
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+
+  gh-release:
+    name: "Github Release"
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: ci-binary-build
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: "Download Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: out/
+
+      - name: "Release"
+        uses: softprops/action-gh-release@v2
+        with:
+          files: "out/**/aptly_*.zip"

.github/workflows/golangci-lint.yml

@@ -0,0 +1,72 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+
+      - name: Install and initialize swagger
+        run: |
+          go install github.com/swaggo/swag/cmd/swag@latest
+          swag init -q --markdownFiles docs
+        shell: sh
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.54.1
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/scripts/upload-artifacts.sh

@@ -0,0 +1,144 @@
+#!/bin/sh
+
+set -e
+
+builds=build/
+packages=${builds}*.deb
+folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
+aptly_user="$APTLY_USER"
+aptly_password="$APTLY_PASSWORD"
+aptly_api="https://aptly-ops.aptly.info"
+version=`make version`
+
+action=$1
+dist=$2
+
+usage() {
+    echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
+    echo "       $0 release" >&2
+}
+
+# repos and publish must be created beforehand:
+#!/bin/sh
+#for dist in buster bullseye bookworm focal jammy noble
+#do
+#        for build in ci release
+#        do
+#            echo
+#            echo "# Creating and publishing $build/$dist"
+#            aptly repo create -distribution=$dist -component=main aptly-$build-$dist
+#            aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
+#                               -distribution=$dist -batch -keyring=aptly.pub \
+#                               aptly-$build-$dist \
+#                               s3:repo.aptly.info:$build
+#       done
+#done
+
+if [ -z "$action" ]; then
+    usage
+    exit 1
+fi
+
+if [ "action" = "ci" ] && [ -z "$dist" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
+    usage
+    echo Error: please set APTLY_USER and APTLY_PASSWORD
+    exit 1
+fi
+
+echo "Publishing version '$version' to $action for $dist...\n"
+
+upload()
+{
+    echo "\nUploading files:"
+    for file in $packages; do
+        echo " - $file"
+        jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+    done
+}
+
+cleanup() {
+    echo "\nCleanup..."
+    jsonret=`curl -fsS -X DELETE  -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+}
+trap cleanup EXIT
+
+update_publish() {
+    _publish=$1
+    _dist=$2
+    jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+        '{"AcquireByHash": true, "MultiDist": true,
+          "Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
+        -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    _success=0
+    for t in `seq 180`
+    do
+        jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_task_id`
+        _state=`echo $jsonret | jq .State`
+        if [ "$_state" = "2" ]; then
+            _success=1
+            curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_task_id
+            break
+        fi
+        if [ "$_state" = "3" ]; then
+            echo Error: publish failed
+            exit 1
+        fi
+        sleep 1
+    done
+    if [ "$_success" -ne 1 ]; then
+        echo "Error: publish failed (timeout)"
+        exit 1
+    fi
+}
+
+if [ "$action" = "ci" ]; then
+    if echo "$version" | grep -vq "+"; then
+       # skip ci when on release tag
+       exit 0
+    fi
+
+    aptly_repository=aptly-ci-$dist
+    aptly_published=s3:repo.aptly.info:ci
+
+elif [ "$action" = "release" ]; then
+    aptly_repository=aptly-release-$dist
+    aptly_published=s3:repo.aptly.info:release
+fi
+
+upload
+
+echo "\nAdding packages to $aptly_repository ..."
+jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder`
+
+echo "\nUpdating published repo $aptly_published ..."
+update_publish $aptly_published $dist
+
+# if [ "$action" = "OBSOLETErelease" ]; then
+#     aptly_repository=aptly-release
+#     aptly_snapshot=aptly-$version
+#     aptly_published=s3:repo.aptly.info:./squeeze
+#
+#     echo "\nAdding packages to $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
+#     echo
+#
+#     echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
+#         "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
+#     echo
+#
+#     echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
+#     curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+#         "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
+#                                    \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
+#                                                  \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
+#         -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
+#     echo
+# fi
+

.gitignore

@@ -2,11 +2,14 @@
 *.o
 *.a
 *.so
+unit.out
 
 # Folders
 _obj
 _test
 
+tmp/
+
 # Architecture specific extensions/prefixes
 *.[568vq]
 [568vq].out
@@ -32,13 +35,36 @@ root/
 man/aptly.1.html
 man/aptly.1.ronn
 
-.goxc.local.json
-
 system/env/
 
 # created by make build for release artifacts
+VERSION
+aptly.test
+
 build/
 
-pgp/keyrings/aptly2*.gpg
+system/files/aptly2.gpg~
-pgp/keyrings/aptly2*.gpg~
+system/files/aptly2_passphrase.gpg~
-pgp/keyrings/.#*
+
+*.creds
+
+.go/
+obj-x86_64-linux-gnu/
+
+# debian
+debian/.debhelper/
+debian/aptly.debhelper.log
+debian/aptly.postrm.debhelper
+debian/aptly.substvars
+debian/aptly/
+debian/debhelper-build-stamp
+debian/files
+debian/aptly-api/
+debian/aptly-api.debhelper.log
+debian/aptly-api.postrm.debhelper
+debian/aptly-api.substvars
+debian/aptly-dbg.debhelper.log
+debian/aptly-dbg.substvars
+debian/aptly-dbg/
+
+docs/

.golangci.yml

@@ -5,7 +5,6 @@ run:
 linters:
   disable-all: true
   enable:
-   - deadcode
    - goconst
    - gofmt
    - goimports
@@ -14,6 +13,4 @@ linters:
    - misspell
    - revive
    - staticcheck
-   - structcheck
-   - varcheck
    - vetshadow

.goxc.json

@@ -1,45 +0,0 @@
-{
-	"AppName": "aptly",
-	"ArtifactsDest": "xc-out/",
-	"TasksExclude": [
-		"rmbin",
-		"go-test",
-		"go-vet"
-	],
-    "TasksAppend": [
-		"bintray"
-    ],
-    "TaskSettings": {
-        "debs": {
-            "metadata": {
-                "maintainer": "Andrey Smirnov",
-                "maintainer-email": "me@smira.ru",
-                "description": "Debian repository management tool"
-            },
-            "metadata-deb": {
-                "License": "MIT",
-                "Homepage": "https://www.aptly.info/",
-                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-				"Suggests": "graphviz"
-            },
-            "other-mapped-files": {
-                "/": "root/"
-            }
-        },
-        "bintray": {
-            "repository": "aptly",
-            "subject": "smira",
-            "package": "aptly",
-            "downloadspage": "bintray.md"
-        }
-    },
-	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
-	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
-	"MainDirsExclude": "_man,vendor",
-	"BuildSettings": {
-		"LdFlagsXVars": {
-			"Version": "main.Version"
-		}
-	},
-	"ConfigVersion": "0.9"
-}

.travis.yml

@@ -1,106 +0,0 @@
-dist: xenial
-sudo: required
-
-language: go
-
-go_import_path: github.com/aptly-dev/aptly
-
-addons:
-  apt:
-    packages:
-    - python-virtualenv
-    - graphviz
-    - gnupg2
-    - gpgv2
-
-env:
-  global:
-    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
-    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
-    - NO_FTP_ACCESS: "yes"
-    - BOTO_CONFIG: /dev/null
-    - GO111MODULE: "on"
-    - GOPROXY: https://proxy.golang.org
-
-matrix:
-  allow_failures:
-    - go: master
-      env: RUN_LONG_TESTS=no
-  fast_finish: true
-  include:
-    - go: 1.11.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.12.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.13.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.14.x
-      env: RUN_LONG_TESTS=yes
-    - go: 1.15.x
-      env:
-        - RUN_LONG_TESTS=yes
-        - DEPLOY_BINARIES=yes
-        - APTLY_USER=aptly
-        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
-    - go: master
-      env: RUN_LONG_TESTS=no
-
-before_install:
-  - virtualenv system/env
-  - . system/env/bin/activate
-  - pip install six packaging appdirs
-  - pip install -U pip setuptools
-  - pip install -r system/requirements.txt
-  - make version
-
-install:
-  - make prepare
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-notifications:
-  webhooks:
-    urls:
-      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false
-
-before_deploy:
-  - make release
-
-deploy:
-  - provider: releases
-    api_key:
-      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
-    file_glob: true
-    file: build/*
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: s3
-    access_key_id:
-      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
-    secret_access_key:
-      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
-    bucket: aptly-nightly
-    skip_cleanup: true
-    acl: public-read
-    local_dir: build
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh nightly
-    skip_cleanup: true
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh release
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -49,3 +49,17 @@ List of contributors, in chronological order:
 * Samuel Mutel (https://github.com/smutel)
 * Russell Greene (https://github.com/russelltg)
 * Wade Simmons (https://github.com/wadey)
+* Steven Stone (https://github.com/smstone)
+* Josh Bayfield (https://github.com/jbayfield)
+* Boxjan (https://github.com/boxjan)
+* Mauro Regli (https://github.com/reglim)
+* Alexander Zubarev (https://github.com/strike)
+* Nicolas Dostert (https://github.com/acdn-ndostert)
+* Ryan Gonzalez (https://github.com/refi64)
+* Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramón N.Rodriguez (https://github.com/runitonmetal)
+* Golf Hu (https://github.com/hudeng-go)
+* Cookie Fei (https://github.com/wuhuang26)

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 :+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
 
-The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
 These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
 ## What should I know before I get started?
@@ -11,7 +11,7 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
 
 This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
 By participating, you are expected to uphold this code.
-Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
 
 ### List of Repositories
 
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your First Code Contribution
+### Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,20 +68,6 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
 Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
 if that is possible.
 
-## Development Setup
-
-This section describes local setup to start contributing to aptly source.
-
-### Go & Python
-
-You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
-
-If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
-initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
-
-Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
-to prepend it or to skip this test if you're security conscious.
-
 ### Forking and Cloning
 
 As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
@@ -98,7 +84,94 @@ to specify your remote name when pushing branches:
 
     git push <user> <your-branch>
 
-### Dependencies
+
+## Development Setup
+
+Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
+
+### Docker Development Setup
+
+This section describes the docker setup to start contributing to aptly.
+
+#### Dependencies
+
+Install the following on your development machine:
+- docker
+- make
+- git
+
+
+#### Create docker container
+
+To build the development docker image, run:
+```
+make docker-image
+```
+
+#### Build aptly
+
+To build the aptly in the development docker container, run:
+```
+make docker-build
+```
+
+#### Running aptly commands
+
+To run aptly commands in the development docker container, run:
+```
+make docker-aptly
+```
+
+Example:
+```
+$ make docker-aptly
+bash: cannot set terminal process group (16): Inappropriate ioctl for device
+bash: no job control in this shell
+aptly@b43e8473ef81:/app$ aptly version
+aptly version: 1.5.0+189+g0fc90dff
+```
+
+#### Running unit tests
+
+In order to run aptly unit tests, enter the following:
+```
+make docker-unit-tests
+```
+
+#### Running system tests
+
+In order to run aptly system tests, enter the following:
+```
+make docker-system-tests
+```
+
+#### Running golangci-lint
+
+In order to run aptly unit tests, run:
+```
+make docker-lint
+```
+
+#### More info
+
+Run `make help` for more information.
+
+
+### Local Development Setup
+
+This section describes local setup to start contributing to aptly.
+
+#### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 3.9 (or newer, the CI currently tests against 3.11).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.
+
+#### Dependencies
 
 You would need some additional tools and Python virtual environment to run tests and checks, install them with:
 
@@ -110,7 +183,7 @@ Aptly is using Go modules to manage dependencies, download modules using:
 
     make modules
 
-### Building
+#### Building
 
 If you want to build aptly binary from your current source tree, run:
 
@@ -124,7 +197,7 @@ Or, if it's not on your path:
 
     ~/go/bin/aptly
 
-### Unit-tests
+#### Unit-tests
 
 aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
 feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -133,7 +206,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
 
     make test
 
-### Functional Tests
+#### Functional Tests
 
 Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
 runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -180,7 +253,7 @@ There are some packages available under `system/files/` directory which are used
 this default location. You can run aptly under different user or by using non-default config location with non-default
 aptly root directory.
 
-### Style Checks
+#### Style Checks
 
 Style checks could be run with:
 
@@ -191,7 +264,7 @@ for the linter could be found in [.golangci.yml](.golangci.yml) file.
 
 Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
 
-### Vendored Code
+#### Vendored Code
 
 aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
 repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
@@ -217,25 +290,3 @@ Bash and Zsh completion for aptly reside in the same repo under in [completion.d
 When new option or command is introduced, bash completion should be updated to reflect that change.
 
 When aptly package is being built, it automatically pulls bash completion and man page into the package.
-
-## Design
-
-This section requires future work.
-
-*TBD*
-
-### Database
-
-### Package Pool
-
-### Package
-
-### PackageList, PackageRefList
-
-### LocalRepo, RemoteRepo, Snapshot
-
-### PublishedRepository
-
-### Context
-
-### Collections, CollectionFactory

Makefile

@@ -1,81 +1,188 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
+GOPATH=$(shell go env GOPATH)
-ifdef TRAVIS_TAG
+VERSION=$(shell make -s version)
-	TAG=$(TRAVIS_TAG)
-else
-	TAG="$(shell git describe --tags --always)"
-endif
-VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python3
-TESTS?=
 BINPATH?=$(GOPATH)/bin
-RUN_LONG_TESTS?=yes
+GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-all: modules test bench check system-test
+# Uncomment to update test outputs
+# CAPTURE := "--capture"
 
-prepare:
+help:  ## Print this help
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-modules:
+prepare:  ## Install go module dependencies
-	go mod download
+	# Prepare go modules
 	go mod verify
 	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-dev:
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
-	go get -u github.com/laher/goxc
+	@reltype=ci ; \
+	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
+	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
+		gittag=`git describe --tags --exact-match 2>/dev/null` ;\
+		if echo "$$gittag" | grep -q '^v[0-9]'; then \
+			reltype=release ; \
+		fi ; \
+	fi ; \
+	echo $$reltype
 
-check: system/env
+version:  ## Print aptly version
-ifeq ($(RUN_LONG_TESTS), yes)
+	@ci="" ; \
-	golangci-lint run
+	if [ "`make -s releasetype`" = "ci" ]; then \
-	system/env/bin/flake8
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
-endif
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
+	else \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
+	fi
+
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest
+
+swagger: swagger-install
+	# Generate swagger docs
+	@PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs
+
+etcd-install:
+	# Install etcd
+	test -d /srv/etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint:
+	# Install golangci-lint
+	go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
 
 install:
-	go install -v -ldflags "-X main.Version=$(VERSION)"
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	go generate
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system/env: system/requirements.txt
+test: prepare swagger etcd-install  ## Run unit tests
-ifeq ($(RUN_LONG_TESTS), yes)
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
-	rm -rf system/env
+	@mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 &
-	$(PYTHON) -m venv system/env
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
-	system/env/bin/pip install -r system/requirements.txt
+	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
-endif
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
 
-system-test: install system/env
+system-test: prepare swagger etcd-install  ## Run system tests
-ifeq ($(RUN_LONG_TESTS), yes)
+	# build coverage binary
+	go test -v -coverpkg="./..." -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
-endif
+	# Run system tests
-
+	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
-test:
-	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
 
 bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
 	go test -v ./deb -run=nothing -bench=. -benchmem
 
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,cmd,systemd -- api serve -listen 0.0.0.0:3142
+
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly>" dch -v `make -s version` "CI build" ; \
+	fi
+	# Run dpkg-buildpackage
+	buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)
+	# cleanup
+	@test -f debian/changelog.dpkg-bak && mv debian/changelog.dpkg-bak debian/changelog || true ; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, MacOS, Linux tar)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-test:  ## Run unit tests in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper test
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper system-test TEST=$(TEST)
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
+
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc: dev
+man:  ## Create man pages
-	rm -rf root/
-	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
-	cp man/aptly.1 root/usr/share/man/man1
-	cp completion.d/aptly root/etc/bash_completion.d/
-	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
-	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
-
-release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
-release: goxc
-	rm -rf build/
-	mkdir -p build/
-	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
-
-man:
 	make -C man
 
-version:
+clean:  ## remove local build and module cache
-	@echo $(VERSION)
+	# Clean all generated and build files
+	test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: man modules version release goxc
+.PHONY: help man prepare swagger version binaries docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image build docker-shell clean releasetype dpkg serve docker-serve flake8

README.rst

@@ -9,7 +9,7 @@ aptly
     :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly

api/api.go

@@ -3,17 +3,18 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
 // Lock order acquisition (canonical):
@@ -27,6 +28,23 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+// GET /api/ready
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+	}
+}
+
+// GET /api/healthy
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
+}
+
 type dbRequestKind int
 
 const (
@@ -137,20 +155,29 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 	// Run this task in background if configured globally or per-request
 	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
 	if background {
-		log.Println("Executing task asynchronously")
+		log.Debug().Msg("Executing task asynchronously")
 		task, conflictErr := runTaskInBackground(name, resources, proc)
 		if conflictErr != nil {
-			c.AbortWithError(409, conflictErr)
+			AbortWithJSONError(c, 409, conflictErr)
 			return
 		}
 		c.JSON(202, task)
 	} else {
-		log.Println("Executing task synchronously")
+		log.Debug().Msg("Executing task synchronously")
-		out := context.Progress()
+		task, conflictErr := runTaskInBackground(name, resources, proc)
-		detail := task.Detail{}
+		if conflictErr != nil {
-		retValue, err := proc(out, &detail)
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
+		context.TaskList().DeleteTaskByID(task.ID)
 		if err != nil {
-			c.AbortWithError(retValue.Code, err)
+			AbortWithJSONError(c, retValue.Code, err)
 			return
 		}
 		if retValue != nil {
@@ -168,7 +195,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -176,7 +203,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -193,7 +220,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
@@ -203,11 +230,41 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false,
+					nil, 0, []string{})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
 		list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
@@ -219,3 +276,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 		c.JSON(200, list.Strings())
 	}
 }
+
+func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+	return c.AbortWithError(code, err)
+}

api/api_test.go

@@ -1,16 +1,20 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
-	ctx "github.com/aptly-dev/aptly/context"
+	"fmt"
-	"github.com/gin-gonic/gin"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
 	"github.com/smira/flag"
 
 	. "gopkg.in/check.v1"
@@ -30,12 +34,13 @@ type ApiSuite struct {
 var _ = Suite(&ApiSuite{})
 
 func createTestConfig() *os.File {
-	file, err := ioutil.TempFile("", "aptly")
+	file, err := os.CreateTemp("", "aptly")
 	if err != nil {
 		return nil
 	}
 	jsonString, err := json.Marshal(gin.H{
-		"architectures": []string{},
+		"architectures":         []string{},
+		"enableMetricsEndpoint": true,
 	})
 	if err != nil {
 		return nil
@@ -44,9 +49,12 @@ func createTestConfig() *os.File {
 	return file
 }
 
-func (s *ApiSuite) SetUpSuite(c *C) {
+func (s *ApiSuite) setupContext() error {
+	aptly.Version = "testVersion"
 	file := createTestConfig()
-	c.Assert(file, NotNil)
+	if nil == file {
+		return fmt.Errorf("unable to create the test configuration file")
+	}
 	s.configFile = file
 
 	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
@@ -57,10 +65,19 @@ func (s *ApiSuite) SetUpSuite(c *C) {
 	s.flags = flags
 
 	context, err := ctx.NewContext(s.flags)
-	c.Assert(err, IsNil)
+	if nil != err {
+		return err
+	}
 
 	s.context = context
 	s.router = Router(context)
+
+	return nil
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	err := s.setupContext()
+	c.Assert(err, IsNil)
 }
 
 func (s *ApiSuite) TearDownSuite(c *C) {
@@ -85,11 +102,52 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
 	return w, nil
 }
 
+func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
+}
+
 func (s *ApiSuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, ".*Version.*")
+	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
+}
+
+func (s *ApiSuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *ApiSuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *ApiSuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
+}
+
+func (s *ApiSuite) TestRepoCreate(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name": "dummy",
+	})
+	c.Assert(err, IsNil)
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
 }
 
 func (s *ApiSuite) TestTruthy(c *C) {

api/error.go

@@ -0,0 +1,5 @@
+package api
+
+type Error struct {
+	Error string `json:"error"`
+}

api/files.go

@@ -6,8 +6,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
 func verifyPath(path string) bool {
@@ -24,27 +27,31 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
 	return true
 }
 
-// GET /files
+// @Summary Get files
+// @Description Get list of uploaded files.
+// @Tags Files
+// @Produce  json
+// @Success 200 {array} string "List of files"
+// @Router /api/files [get]
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
-		if err != nil {
-			return err
-		}
-
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -53,7 +60,7 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -66,17 +73,17 @@ func apiFilesUpload(c *gin.Context) {
 		return
 	}
 
-	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -86,7 +93,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +101,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -109,6 +116,7 @@ func apiFilesUpload(c *gin.Context) {
 		}
 	}
 
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
 
 }
@@ -120,9 +128,10 @@ func apiFilesListFiles(c *gin.Context) {
 	}
 
 	list := []string{}
-	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	listLock := &sync.Mutex{}
+	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -131,6 +140,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil
@@ -138,9 +149,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 		} else {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 		}
 		return
 	}
@@ -154,9 +165,9 @@ func apiFilesDeleteDir(c *gin.Context) {
 		return
 	}
 
-	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -169,15 +180,17 @@ func apiFilesDeleteFile(c *gin.Context) {
 		return
 	}
 
-	if !verifyPath(c.Params.ByName("name")) {
+	dir := utils.SanitizePath(c.Params.ByName("dir"))
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+	name := utils.SanitizePath(c.Params.ByName("name"))
+	if !verifyPath(name) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 	}

api/gpg.go

@@ -2,13 +2,13 @@ package api
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
 
@@ -24,9 +24,13 @@ func apiGPGAddKey(c *gin.Context) {
 	if c.Bind(&b) != nil {
 		return
 	}
+	b.Keyserver = utils.SanitizePath(b.Keyserver)
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
+	// b.Keyring can be an absolute path
 
 	var err error
-	args := []string{"--no-default-keyring"}
+	args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
 	keyring := "trustedkeys.gpg"
 	if len(b.Keyring) > 0 {
 		keyring = b.Keyring
@@ -37,9 +41,9 @@ func apiGPGAddKey(c *gin.Context) {
 	}
 	if len(b.GpgKeyArmor) > 0 {
 		var tempdir string
-		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 		defer os.RemoveAll(tempdir)
@@ -47,11 +51,11 @@ func apiGPGAddKey(c *gin.Context) {
 		keypath := filepath.Join(tempdir, "key")
 		keyfile, e := os.Create(keypath)
 		if e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 			return
 		}
 		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 		}
 		args = append(args, "--import", keypath)
 
@@ -62,10 +66,10 @@ func apiGPGAddKey(c *gin.Context) {
 		args = append(args, keys...)
 	}
 
-	finder := pgp.GPG1Finder()
+	finder := pgp.GPGDefaultFinder()
 	gpg, _, err := finder.FindGPG()
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -74,11 +78,11 @@ func apiGPGAddKey(c *gin.Context) {
 	// there is no error handling for such as gpg will do this for us
 	cmd := exec.Command(gpg, args...)
 	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
-	cmd.Stdout = os.Stdout
+	out, err := cmd.CombinedOutput()
-	if err = cmd.Run(); err != nil {
+	if err != nil {
-		c.AbortWithError(400, err)
+		c.JSON(400, string(out))
 		return
 	}
 
-	c.JSON(200, gin.H{})
+	c.JSON(200, string(out))
 }

api/graph.go

@@ -43,25 +43,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/metrics.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+	apiFilesUploadedCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_files_uploaded_total",
+			Help: "Total number of uploaded files labeled by upload directory.",
+		},
+		[]string{"directory"},
+	)
+	apiReposPackageCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_repos_package_count",
+			Help: "Current number of published packages labeled by source, distribution and component.",
+		},
+		[]string{"source", "distribution", "component"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
+
+func countPackagesByRepos() {
+	err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if err != nil {
+			msg := fmt.Sprintf(
+				"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
+				err, repo.StoragePrefix(), repo.Distribution, repo.Components())
+			log.Warn().Msg(msg)
+			return err
+		}
+
+		components := repo.Components()
+		for _, c := range components {
+			count := float64(len(repo.RefList(c).Refs))
+			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
+		log.Warn().Msg(msg)
+	}
+}

api/middleware.go

@@ -9,59 +9,36 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
-)
-
-var (
-	apiRequestsInFlightGauge = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "aptly_api_http_requests_in_flight",
-			Help: "Number of concurrent HTTP api requests currently handled.",
-		},
-		[]string{"method", "path"},
-	)
-	apiRequestsTotalCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "aptly_api_http_requests_total",
-			Help: "Total number of api requests.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_size_bytes",
-			Help: "Api HTTP request size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiResponseSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_response_size_bytes",
-			Help: "Api HTTP response size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestsDurationSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_duration_seconds",
-			Help: "Duration of api requests in seconds.",
-		},
-		[]string{"code", "method", "path"},
-	)
 )
 
 // Only use base path as label value (e.g.: /api/repos) because of time series cardinality
 // See https://prometheus.io/docs/practices/naming/#labels
 func getBasePath(c *gin.Context) string {
-	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+	segment0, err := getURLSegment(c.Request.URL.Path, 0)
+	if err != nil {
+		return "/"
+	}
+	segment1, err := getURLSegment(c.Request.URL.Path, 1)
+	if err != nil {
+		return *segment0
+	}
+
+	return *segment0 + *segment1
 }
 
-func getURLSegment(url string, idx int) string {
+func getURLSegment(url string, idx int) (*string, error) {
-	var urlSegments = strings.Split(url, "/")
+	urlSegments := strings.Split(url, "/")
-
 	// Remove segment at index 0 because it's an empty string
-	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
+	urlSegments = urlSegments[1:cap(urlSegments)]
-	return fmt.Sprintf("/%s", segmentAtIndex)
+
+	if len(urlSegments) <= idx {
+		return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
+	}
+
+	segmentAtIndex := urlSegments[idx]
+	s := fmt.Sprintf("/%s", segmentAtIndex)
+	return &s, nil
 }
 
 func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
@@ -101,3 +78,39 @@ func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Co
 		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
 	}
 }
+
+// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
+// logs that include error messages if there are any.
+func JSONLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
+
+		// Process request
+		c.Next()
+
+		ts := time.Now()
+		if raw != "" {
+			path = path + "?" + raw
+		}
+
+		errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
+		l := log.With().Str("remote", c.ClientIP()).Logger().
+			With().Str("method", c.Request.Method).Logger().
+			With().Str("path", path).Logger().
+			With().Str("protocol", c.Request.Proto).Logger().
+			With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
+			With().Str("latency", ts.Sub(start).String()).Logger().
+			With().Str("agent", c.Request.UserAgent()).Logger()
+
+		if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
+			l.Warn().Msg(errorMessage)
+		} else if c.Writer.Status() >= 500 {
+			l.Error().Msg(errorMessage)
+		} else {
+			l.Info().Msg(errorMessage)
+		}
+	}
+}

api/middleware_test.go

@@ -0,0 +1,255 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MiddlewareSuite struct {
+	router    http.Handler
+	context   *gin.Context
+	logReader *os.File
+	logWriter *os.File
+}
+
+var _ = Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) SetUpTest(c *C) {
+	r, w, err := os.Pipe()
+	c.Assert(err, IsNil)
+
+	utils.SetupJSONLogger("debug", w)
+	mw := JSONLogger()
+
+	router := gin.New()
+	router.UseRawPath = true
+	router.Use(mw)
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	root := router.Group("/api")
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	root.GET("/ready", apiReady(isReady))
+	root.GET("/healthy", apiHealthy)
+
+	s.router = router
+	s.logReader = r
+	s.logWriter = w
+}
+
+func (s *MiddlewareSuite) TearDownTest(c *C) {
+	s.router = nil
+	s.context = nil
+	s.logReader = nil
+	s.logWriter = nil
+}
+
+func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
+	recorder := httptest.NewRecorder()
+	s.context, _ = gin.CreateTestContext(recorder)
+	req, _ := http.NewRequestWithContext(s.context, method, url, body)
+	s.context.Request = req
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "warn")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["method"]; ok {
+		c.Check(val, Equals, "GET")
+	} else {
+		c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["path"]; ok {
+		c.Check(val, Equals, "/")
+	} else {
+		c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["protocol"]; ok {
+		c.Check(val, Equals, "HTTP/1.1")
+	} else {
+		c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["code"]; ok {
+		c.Check(val, Equals, "404")
+	} else {
+		c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["remote"]; !ok {
+		c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["latency"]; !ok {
+		c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["agent"]; !ok {
+		c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["time"]; !ok {
+		c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "error")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	fmt.Println(capturedOutput)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestGetBasePath(c *C) {
+	s.HTTPRequest(http.MethodGet, "", nil)
+	path := getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/api", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api")
+
+	s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api/repos")
+}
+
+func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
+	url := "/"
+	segment, err := getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api/repos/testRepo"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	segment, err = getURLSegment(url, 1)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/repos")
+}

api/mirror.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
+	"os"
 	"sort"
 	"strings"
 	"sync"
@@ -14,19 +14,16 @@ import (
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
-func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
+func getVerifier(keyRings []string) (pgp.Verifier, error) {
-	if ignoreSignatures {
-		return nil, nil
-	}
-
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(false)
 	if err != nil {
 		return nil, err
 	}
@@ -34,7 +31,13 @@ func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error)
 	return verifier, nil
 }
 
-// GET /api/mirrors
+// @Summary Get mirrors
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {array} deb.RemoteRepo
+// @Router /api/mirrors [get]
 func apiMirrorsList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -48,24 +51,49 @@ func apiMirrorsList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// POST /api/mirrors
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
+// @Summary Create mirror
+// @Description **Create a mirror**
+// @Tags Mirrors
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/mirrors [post]
 func apiMirrorsCreate(c *gin.Context) {
 	var err error
-	var b struct {
+	var b mirrorCreateParams
-		Name               string `binding:"required"`
-		ArchiveURL         string `binding:"required"`
-		Distribution       string
-		Filter             string
-		Components         []string
-		Architectures      []string
-		Keyrings           []string
-		DownloadSources    bool
-		DownloadUdebs      bool
-		DownloadInstaller  bool
-		FilterWithDeps     bool
-		SkipComponentCheck bool
-		IgnoreSignatures   bool
-	}
 
 	b.DownloadSources = context.Config().DownloadSourcePackages
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -81,7 +109,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
 		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 	}
@@ -89,7 +117,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	if b.Filter != "" {
 		_, err = query.Parse(b.Filter)
 		if err != nil {
-			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 			return
 		}
 	}
@@ -98,39 +126,50 @@ func apiMirrorsCreate(c *gin.Context) {
 		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
 
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 		return
 	}
 
 	repo.Filter = b.Filter
 	repo.FilterWithDeps = b.FilterWithDeps
 	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.SkipArchitectureCheck = b.SkipArchitectureCheck
 	repo.DownloadSources = b.DownloadSources
 	repo.DownloadUdebs = b.DownloadUdebs
 
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
 	downloader := context.NewDownloader(nil)
-	err = repo.Fetch(downloader, verifier)
+	err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
 		return
 	}
 
 	err = collection.Add(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
 		return
 	}
 
 	c.JSON(201, repo)
 }
 
-// DELETE /api/mirrors/:name
+// @Summary Delete Mirror
+// @Description **Delete a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param force query int true "force: 1 to enable"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
+// @Failure 500 {object} Error "Unable to delete"
+// @Router /api/mirrors/{name} [delete]
 func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
@@ -141,13 +180,13 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
@@ -157,7 +196,7 @@ func apiMirrorsDrop(c *gin.Context) {
 			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
@@ -169,7 +208,15 @@ func apiMirrorsDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/mirrors/:name
+// @Summary Show Mirror
+// @Description **Get mirror information by name**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [get]
 func apiMirrorsShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -177,19 +224,30 @@ func apiMirrorsShow(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
 
 	c.JSON(200, repo)
 }
 
-// GET /api/mirrors/:name/packages
+// @Summary List Mirror Packages
+// @Description **Get a list of packages from a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Produce json
+// @Success 200 {array} deb.Package "List of Packages"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name}/packages [get]
 func apiMirrorsPackages(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -197,17 +255,17 @@ func apiMirrorsPackages(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
 
 	if repo.LastDownloadDate.IsZero() {
-		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
 		return
 	}
 
@@ -216,7 +274,7 @@ func apiMirrorsPackages(c *gin.Context) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -224,7 +282,7 @@ func apiMirrorsPackages(c *gin.Context) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -241,7 +299,7 @@ func apiMirrorsPackages(c *gin.Context) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
@@ -251,7 +309,7 @@ func apiMirrorsPackages(c *gin.Context) {
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 		}
 	}
 
@@ -267,36 +325,65 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 }
 
-// PUT /api/mirrors/:name
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Url of the archive to mirror
+	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
+	// Package query that is applied to mirror packages
+	Filter string `               json:"Filter"                 example:"xserver-xorg"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `      json:"Architectures"          example:"amd64"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `         json:"Components"             example:"main"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `         json:"FilterWithDeps"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `        json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `          json:"DownloadUdebs"`
+	// Set "true" to skip checking if the given components are in the Release file
+	SkipComponentCheck bool `     json:"SkipComponentCheck"`
+	// Set "true" to skip checking if the given architectures are in the Release file
+	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+}
+
+// @Summary Update Mirror
+// @Description **Update Mirror and download packages**
+// @Tags Mirrors
+// @Param name path string true "mirror name to update"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [put]
 func apiMirrorsUpdate(c *gin.Context) {
 	var (
 		err    error
 		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
 	)
 
-	var b struct {
-		Name                 string
-		ArchiveURL           string
-		Filter               string
-		Architectures        []string
-		Components           []string
-		Keyrings             []string
-		FilterWithDeps       bool
-		DownloadSources      bool
-		DownloadUdebs        bool
-		SkipComponentCheck   bool
-		IgnoreChecksums      bool
-		IgnoreSignatures     bool
-		ForceUpdate          bool
-		SkipExistingPackages bool
-	}
-
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
 	remote, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -304,12 +391,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	b.DownloadUdebs = remote.DownloadUdebs
 	b.DownloadSources = remote.DownloadSources
 	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
 	b.FilterWithDeps = remote.FilterWithDeps
 	b.Filter = remote.Filter
 	b.Architectures = remote.Architectures
 	b.Components = remote.Components
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
-	log.Printf("%s: Starting mirror update\n", b.Name)
+	log.Info().Msgf("%s: Starting mirror update", b.Name)
 
 	if c.Bind(&b) != nil {
 		return
@@ -318,14 +407,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
-			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
 			return
 		}
 	}
 
 	if b.DownloadUdebs != remote.DownloadUdebs {
 		if remote.IsFlat() && b.DownloadUdebs {
-			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
 			return
 		}
 	}
@@ -338,14 +427,15 @@ func apiMirrorsUpdate(c *gin.Context) {
 	remote.DownloadUdebs = b.DownloadUdebs
 	remote.DownloadSources = b.DownloadSources
 	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
 	remote.FilterWithDeps = b.FilterWithDeps
 	remote.Filter = b.Filter
 	remote.Architectures = b.Architectures
 	remote.Components = b.Components
 
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
@@ -353,7 +443,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier)
+		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -365,7 +455,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -458,7 +548,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}()
 
-		log.Printf("%s: Spawning background processes...\n", b.Name)
+		log.Info().Msgf("%s: Spawning background processes...", b.Name)
 		var wg sync.WaitGroup
 		for i := 0; i < context.Config().DownloadConcurrency; i++ {
 			wg.Add(1)
@@ -476,7 +566,16 @@ func apiMirrorsUpdate(c *gin.Context) {
 						var e error
 
 						// provision download location
-						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+							task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+						} else {
+							var file *os.File
+							file, e = os.CreateTemp("", task.File.Filename)
+							if e == nil {
+								task.TempDownPath = file.Name()
+								file.Close()
+							}
+						}
 						if e != nil {
 							pushError(e)
 							continue
@@ -494,6 +593,20 @@ func apiMirrorsUpdate(c *gin.Context) {
 							continue
 						}
 
+						// and import it back to the pool
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						if err != nil {
+							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+							pushError(err)
+							continue
+						}
+
+						// update "attached" files if any
+						for _, additionalAtask := range task.Additional {
+							additionalAtask.File.PoolPath = task.File.PoolPath
+							additionalAtask.File.Checksums = task.File.Checksums
+						}
+
 						task.Done = true
 						taskFinished <- task
 					case <-context.Done():
@@ -505,32 +618,22 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		// Wait for all download goroutines to finish
-		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
 		wg.Wait()
-		log.Printf("%s: Background processes finished\n", b.Name)
+		log.Info().Msgf("%s: Background processes finished", b.Name)
 		close(taskFinished)
 
-		for idx := range queue {
+		defer func() {
+			for _, task := range queue {
+				if task.TempDownPath == "" {
+					continue
+				}
 
-			atask := &queue[idx]
+				if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
-
+					fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
-			if !atask.Done {
+				}
-				// download not finished yet
-				continue
 			}
-
+		}()
-			// and import it back to the pool
-			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
-			}
-
-			// update "attached" files if any
-			for _, additionalAtask := range atask.Additional {
-				additionalAtask.File.PoolPath = atask.File.PoolPath
-				additionalAtask.File.Checksums = atask.File.Checksums
-			}
-		}
 
 		select {
 		case <-context.Done():
@@ -539,18 +642,18 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		if len(errors) > 0 {
-			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 		}
 
-		log.Printf("%s: Finalizing download\n", b.Name)
+		log.Info().Msgf("%s: Finalizing download...", b.Name)
 		remote.FinalizeDownload(collectionFactory, out)
 		err = collectionFactory.RemoteRepoCollection().Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		log.Info().Msgf("%s: Mirror updated successfully", b.Name)
 		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
 	})
 }

api/packages.go

@@ -9,9 +9,24 @@ func apiPackagesShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, p)
 }
+
+// @Summary Get packages
+// @Description Get list of packages.
+// @Tags Packages
+// @Consume  json
+// @Produce  json
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Success 200 {array} string "List of packages"
+// @Router /api/packages [get]
+func apiPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PackageCollection()
+	showPackages(c, collection.AllPackageRefs(), collectionFactory)
+}

api/packages_test.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type PackagesSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&PackagesSuite{})
+
+func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}

api/publish.go

@@ -16,7 +16,6 @@ import (
 // SigningOptions is a shared between publish API GPG options structure
 type SigningOptions struct {
 	Skip           bool
-	Batch          bool
 	GpgKey         string
 	Keyring        string
 	SecretKeyring  string
@@ -33,7 +32,9 @@ func getSigner(options *SigningOptions) (pgp.Signer, error) {
 	signer.SetKey(options.GpgKey)
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
-	signer.SetBatch(options.Batch)
+
+	// If Batch is false, GPG will ask for passphrase on stdin, which would block the api process
+	signer.SetBatch(true)
 
 	err := signer.Init()
 	if err != nil {
@@ -43,16 +44,23 @@ func getSigner(options *SigningOptions) (pgp.Signer, error) {
 	return signer, nil
 }
 
-// Replace '_' with '/' and double '__' with single '_'
+// Replace '_' with '/' and double '__' with single '_', SanitizePath
-func parseEscapedPath(path string) string {
+func slashEscape(path string) string {
 	result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
+	result = utils.SanitizePath(result)
 	if result == "" {
 		result = "."
 	}
 	return result
 }
 
-// GET /publish
+// @Summary Get publish points
+// @Description Get list of available publish points. Each publish point is returned as in “show” API.
+// @Tags Publish
+// @Produce  json
+// @Success 200 {array} deb.PublishedRepo
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/publish [get]
 func apiPublishList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
@@ -60,7 +68,7 @@ func apiPublishList(c *gin.Context) {
 	result := make([]*deb.PublishedRepo, 0, collection.Len())
 
 	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
-		err := collection.LoadComplete(repo, collectionFactory)
+		err := collection.LoadShallow(repo, collectionFactory)
 		if err != nil {
 			return err
 		}
@@ -71,7 +79,7 @@ func apiPublishList(c *gin.Context) {
 	})
 
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -80,7 +88,7 @@ func apiPublishList(c *gin.Context) {
 
 // POST /publish/:prefix
 func apiPublishRepoOrSnapshot(c *gin.Context) {
-	param := parseEscapedPath(c.Params.ByName("prefix"))
+	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 
 	var b struct {
@@ -100,20 +108,23 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Architectures        []string
 		Signing              SigningOptions
 		AcquireByHash        *bool
+		MultiDist            bool
 	}
 
 	if c.Bind(&b) != nil {
 		return
 	}
 
+	b.Distribution = utils.SanitizePath(b.Distribution)
+
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
 	if len(b.Sources) == 0 {
-		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to publish: soures are empty"))
 		return
 	}
 
@@ -123,7 +134,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	var resources []string
 	collectionFactory := context.NewCollectionFactory()
 
-	if b.SourceKind == "snapshot" {
+	if b.SourceKind == deb.SourceSnapshot {
 		var snapshot *deb.Snapshot
 
 		snapshotCollection := collectionFactory.SnapshotCollection()
@@ -134,14 +145,14 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			snapshot, err = snapshotCollection.ByName(source.Name)
 			if err != nil {
-				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			resources = append(resources, string(snapshot.ResourceKey()))
 			err = snapshotCollection.LoadComplete(snapshot)
 			if err != nil {
-				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+				AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
@@ -158,26 +169,26 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			localRepo, err = localCollection.ByName(source.Name)
 			if err != nil {
-				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
+				AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			resources = append(resources, string(localRepo.Key()))
 			err = localCollection.LoadComplete(localRepo)
 			if err != nil {
-				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+				AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
 			}
 
 			sources = append(sources, localRepo)
 		}
 	} else {
-		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
+		AbortWithJSONError(c, 400, fmt.Errorf("unknown SourceKind"))
 		return
 	}
 
-	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory)
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, b.MultiDist)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 
@@ -211,7 +222,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		}
 
 		published.SkipBz2 = context.Config().SkipBz2Publishing
-		if b.SkipContents != nil {
+		if b.SkipBz2 != nil {
 			published.SkipBz2 = *b.SkipBz2
 		}
 
@@ -241,9 +252,9 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 // PUT /publish/:prefix/:distribution
 func apiPublishUpdateSwitch(c *gin.Context) {
-	param := parseEscapedPath(c.Params.ByName("prefix"))
+	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
-	distribution := c.Params.ByName("distribution")
+	distribution := utils.SanitizePath(c.Params.ByName("distribution"))
 
 	var b struct {
 		ForceOverwrite bool
@@ -256,6 +267,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			Name      string `binding:"required"`
 		}
 		AcquireByHash *bool
+		MultiDist     *bool
 	}
 
 	if c.Bind(&b) != nil {
@@ -264,7 +276,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
@@ -273,12 +285,12 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 	err = collection.LoadComplete(published, collectionFactory)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
@@ -288,7 +300,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
-			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			AbortWithJSONError(c, 400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
 		updatedComponents = published.Components()
@@ -296,23 +308,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			published.UpdateLocalRepo(component)
 		}
 	} else if published.SourceKind == "snapshot" {
-		publishedComponents := published.Components()
 		for _, snapshotInfo := range b.Snapshots {
-			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
-				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
-				return
-			}
-
 			snapshotCollection := collectionFactory.SnapshotCollection()
 			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err2 != nil {
-				c.AbortWithError(404, err2)
+				AbortWithJSONError(c, 404, err2)
 				return
 			}
 
 			err2 = snapshotCollection.LoadComplete(snapshot)
 			if err2 != nil {
-				c.AbortWithError(500, err2)
+				AbortWithJSONError(c, 500, err2)
 				return
 			}
 
@@ -321,7 +327,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			updatedSnapshots = append(updatedSnapshots, snapshot.Name)
 		}
 	} else {
-		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
+		AbortWithJSONError(c, 500, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
@@ -337,9 +343,13 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.AcquireByHash = *b.AcquireByHash
 	}
 
+	if b.MultiDist != nil {
+		published.MultiDist = *b.MultiDist
+	}
+
 	resources = append(resources, string(published.Key()))
 	taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
@@ -367,7 +377,7 @@ func apiPublishDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
 
-	param := parseEscapedPath(c.Params.ByName("prefix"))
+	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := c.Params.ByName("distribution")
 
@@ -376,14 +386,14 @@ func apiPublishDrop(c *gin.Context) {
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("unable to drop: %s", err))
+		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 
 	resources := []string{string(published.Key())}
 
 	taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := collection.Remove(context, storage, prefix, distribution,
 			collectionFactory, out, force, skipCleanup)
 		if err != nil {

api/repos.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -17,7 +18,44 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/repos
+// GET /repos
+func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+		c.Writer.Flush()
+		c.Writer.WriteString("<pre>\n")
+		if len(localRepos) == 0 {
+			c.Writer.WriteString("<a href=\"-/\">default</a>\n")
+		}
+		for publishPrefix := range localRepos {
+			c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
+		}
+		c.Writer.WriteString("</pre>")
+		c.Writer.Flush()
+	}
+}
+
+// GET /repos/:storage/*pkgPath
+func reposServeInAPIMode(c *gin.Context) {
+	pkgpath := c.Param("pkgPath")
+
+	storage := c.Param("storage")
+	if storage == "-" {
+		storage = ""
+	} else {
+		storage = "filesystem:" + storage
+	}
+
+	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	c.FileFromFS(pkgpath, http.Dir(publicPath))
+}
+
+// @Summary Get repos
+// @Description Get list of available repos. Each repo is returned as in “show” API.
+// @Tags Repos
+// @Produce  json
+// @Success 200 {array} deb.LocalRepo
+// @Router /api/repos [get]
 func apiReposList(c *gin.Context) {
 	result := []*deb.LocalRepo{}
 
@@ -31,7 +69,18 @@ func apiReposList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// POST /api/repos
+// @Summary Create repository
+// @Description Create a local repository.
+// @Tags Repos
+// @Produce  json
+// @Consume  json
+// @Param Name query string false "Name of repository to be created."
+// @Param Comment query string false "Text describing local repository, for the user"
+// @Param DefaultDistribution query string false "Default distribution when publishing from this local repo"
+// @Param DefaultComponent query string false "Default component when publishing from this local repo"
+// @Success 201 {object} deb.LocalRepo
+// @Failure 400 {object} Error "Repository already exists"
+// @Router /api/repos [post]
 func apiReposCreate(c *gin.Context) {
 	var b struct {
 		Name                string `binding:"required"`
@@ -52,7 +101,7 @@ func apiReposCreate(c *gin.Context) {
 	collection := collectionFactory.LocalRepoCollection()
 	err := collection.Add(repo)
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -77,7 +126,7 @@ func apiReposEdit(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -85,7 +134,7 @@ func apiReposEdit(c *gin.Context) {
 		_, err := collection.ByName(*b.Name)
 		if err == nil {
 			// already exists
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 		repo.Name = *b.Name
@@ -102,7 +151,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -110,13 +159,21 @@ func apiReposEdit(c *gin.Context) {
 }
 
 // GET /api/repos/:name
+// @Summary Get repository info by name
+// @Description Returns basic information about local repository.
+// @Tags Repos
+// @Produce  json
+// @Param name path string true "Repository name"
+// @Success 200 {object} deb.LocalRepo
+// @Failure 404 {object} Error "Repository not found"
+// @Router /api/repos/{name} [get]
 func apiReposShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -135,13 +192,13 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
@@ -165,13 +222,13 @@ func apiReposPackagesShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -193,18 +250,18 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
-	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		out.Printf("Loading packages...\n")
 		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
 		if err != nil {
@@ -262,7 +319,22 @@ func apiReposPackageFromFile(c *gin.Context) {
 	apiReposPackageFromDir(c)
 }
 
-// POST /repos/:name/file/:dir
+// @Summary Add packages from uploaded file/directory
+// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
+// @Description Adding same package to local repository is not an error.
+// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory to add"
+// @Consume  json
+// @Param noRemove query string false "when value is set to 1, don’t remove any files"
+// @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)"
+// @Produce  json
+// @Success 200 {string} string "OK"
+// @Failure 400 {object} Error "wrong file"
+// @Failure 404 {object} Error "Repository not found"
+// @Failure 500 {object} Error "Error adding files"
+// @Router /api/repos/{name}/{dir} [post]
 func apiReposPackageFromDir(c *gin.Context) {
 	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
 	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
@@ -271,10 +343,10 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	dirParam := c.Params.ByName("dir")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
-	fileParam := c.Params.ByName("file")
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -284,13 +356,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -306,7 +378,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		verifier := context.GetVerifier()
 
 		var (
@@ -382,6 +454,150 @@ func apiReposPackageFromDir(c *gin.Context) {
 	})
 }
 
+// POST /repos/:name/copy/:src/:file
+func apiReposCopyPackage(c *gin.Context) {
+	dstRepoName := c.Params.ByName("name")
+	srcRepoName := c.Params.ByName("src")
+	fileName := c.Params.ByName("file")
+
+	jsonBody := struct {
+		WithDeps bool `json:"with-deps,omitempty"`
+		DryRun   bool `json:"dry-run,omitempty"`
+	}{
+		WithDeps: false,
+		DryRun:   false,
+	}
+
+	err := c.Bind(&jsonBody)
+	if err != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
+		return
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
+		return
+	}
+
+	var (
+		srcRefList *deb.PackageRefList
+		srcRepo    *deb.LocalRepo
+	)
+
+	srcRepo, err = collectionFactory.LocalRepoCollection().ByName(srcRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
+		return
+	}
+
+	if srcRepo.UUID == dstRepo.UUID {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest and source are identical"))
+		return
+	}
+
+	err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
+		return
+	}
+
+	srcRefList = srcRepo.RefList()
+	taskName := fmt.Sprintf("Copy packages from repo %s to repo %s", srcRepoName, dstRepoName)
+	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		reporter := &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
+
+		}
+
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
+		}
+
+		srcList.PrepareIndex()
+
+		var architecturesList []string
+
+		if jsonBody.WithDeps {
+			dstList.PrepareIndex()
+
+			// Calculate architectures
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = dstList.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			}
+		}
+
+		// srcList.Filter|FilterWithProgress only accept query list
+		queries := make([]deb.PackageQuery, 1)
+		queries[0], err = query.Parse(fileName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
+		}
+
+		toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
+		}
+
+		if toProcess.Len() == 0 {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("no package found for filter: '%s'", fileName)
+		}
+
+		err = toProcess.ForEach(func(p *deb.Package) error {
+			err = dstList.Add(p)
+			if err != nil {
+				return err
+			}
+
+			name := fmt.Sprintf("added %s-%s(%s)", p.Name, p.Version, p.Architecture)
+			reporter.AddedLines = append(reporter.AddedLines, name)
+			return nil
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("error processing dest add: %s", err)
+		}
+
+		if jsonBody.DryRun {
+			reporter.Warning("Changes not saved, as dry run has been requested")
+		} else {
+			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
+
+			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report": reporter,
+		}}, nil
+	})
+}
+
 // POST /repos/:name/include/:dir/:file
 func apiReposIncludePackageFromFile(c *gin.Context) {
 	// redirect all work to dir method
@@ -404,10 +620,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	var sources []string
 	var taskName string
-	dirParam := c.Params.ByName("dir")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
-	fileParam := c.Params.ByName("file")
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -421,7 +637,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("error parsing repo template: %s", err))
 		return
 	}
 
@@ -432,7 +648,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		// repo template string is simple text so only use resource key of specific repository
 		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
 		if err != nil {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
@@ -440,7 +656,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	}
 	resources = append(resources, sources...)
 
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()

api/router.go

@@ -2,36 +2,92 @@ package api
 
 import (
 	"net/http"
+	"os"
+	"sync/atomic"
 
+	"github.com/aptly-dev/aptly/aptly"
 	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog/log"
+
+	_ "github.com/aptly-dev/aptly/docs" // import docs
+	swaggerFiles "github.com/swaggo/files"
+	ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
+		countPackagesByRepos()
 		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
 	}
 }
 
+func redirectSwagger(c *gin.Context) {
+	if c.Request.URL.Path == "/docs/" {
+		c.Redirect(http.StatusMovedPermanently, "/docs/index.html")
+		return
+	}
+	c.Next()
+}
+
 // Router returns prebuilt with routes http.Handler
+// @title           Aptly API
+// @version         1.0
+// @description     Aptly REST API Documentation
+
+// @contact.name   Aptly
+// @contact.url    http://github.com/aptly-dev/aptly
+// @contact.email  support@aptly.info
+
+// @license.name  MIT License
+// @license.url   http://www.
+
+// @BasePath  /api
 func Router(c *ctx.AptlyContext) http.Handler {
-	context = c
+	if aptly.EnableDebug {
-
+		gin.SetMode(gin.DebugMode)
-	router := gin.Default()
+	} else {
-	router.UseRawPath = true
+		gin.SetMode(gin.ReleaseMode)
-	router.Use(gin.ErrorLogger())
-
-	if c.Config().EnableMetricsEndpoint {
-		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
 	}
 
+	router := gin.New()
+	context = c
+
+	router.UseRawPath = true
+
+	if c.Config().LogFormat == "json" {
+		c.StructuredLogging(true)
+		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
+		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
+		router.Use(JSONLogger())
+	} else {
+		c.StructuredLogging(false)
+		utils.SetupDefaultLogger(c.Config().LogLevel)
+		router.Use(gin.Logger())
+	}
+
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	if c.Config().EnableSwaggerEndpoint {
+		router.Use(redirectSwagger)
+		url := ginSwagger.URL("/docs/doc.json")
+		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	}
+
+	if c.Config().EnableMetricsEndpoint {
+		MetricsCollectorRegistrar.Register(router)
+	}
+
+	if c.Config().ServeInAPIMode {
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
+		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
+	}
+
+	api := router.Group("/api")
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
@@ -40,7 +96,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 		go acquireDatabase()
 
-		router.Use(func(c *gin.Context) {
+		api.Use(func(c *gin.Context) {
 			var err error
 
 			errCh := make(chan error)
@@ -48,7 +104,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			err = <-errCh
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -56,7 +112,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 				dbRequests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.AbortWithError(500, err)
+					AbortWithJSONError(c, 500, err)
 				}
 			}()
 
@@ -64,99 +120,113 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		})
 	}
 
-	root := router.Group("/api")
-
 	{
 		if c.Config().EnableMetricsEndpoint {
-			root.GET("/metrics", apiMetricsGet())
+			api.GET("/metrics", apiMetricsGet())
 		}
-		root.GET("/version", apiVersion)
+		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
+
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		api.GET("/ready", apiReady(isReady))
+		api.GET("/healthy", apiHealthy)
 	}
 
 	{
-		root.GET("/repos", apiReposList)
+		api.GET("/repos", apiReposList)
-		root.POST("/repos", apiReposCreate)
+		api.POST("/repos", apiReposCreate)
-		root.GET("/repos/:name", apiReposShow)
+		api.GET("/repos/:name", apiReposShow)
-		root.PUT("/repos/:name", apiReposEdit)
+		api.PUT("/repos/:name", apiReposEdit)
-		root.DELETE("/repos/:name", apiReposDrop)
+		api.DELETE("/repos/:name", apiReposDrop)
 
-		root.GET("/repos/:name/packages", apiReposPackagesShow)
+		api.GET("/repos/:name/packages", apiReposPackagesShow)
-		root.POST("/repos/:name/packages", apiReposPackagesAdd)
+		api.POST("/repos/:name/packages", apiReposPackagesAdd)
-		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+		api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
 
-		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
-		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
 
-		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
-		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+		api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
 
-		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+		api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 
 	{
-		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+		api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
 	{
-		root.GET("/mirrors", apiMirrorsList)
+		api.GET("/mirrors", apiMirrorsList)
-		root.GET("/mirrors/:name", apiMirrorsShow)
+		api.GET("/mirrors/:name", apiMirrorsShow)
-		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
-		root.POST("/mirrors", apiMirrorsCreate)
+		api.POST("/mirrors", apiMirrorsCreate)
-		root.PUT("/mirrors/:name", apiMirrorsUpdate)
+		api.PUT("/mirrors/:name", apiMirrorsUpdate)
-		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
-		root.POST("/gpg/key", apiGPGAddKey)
+		api.POST("/gpg/key", apiGPGAddKey)
 	}
 
 	{
-		root.GET("/files", apiFilesListDirs)
+		api.GET("/s3", apiS3List)
-		root.POST("/files/:dir", apiFilesUpload)
-		root.GET("/files/:dir", apiFilesListFiles)
-		root.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
 	}
 
 	{
-		root.GET("/publish", apiPublishList)
+		api.GET("/files", apiFilesListDirs)
-		root.POST("/publish", apiPublishRepoOrSnapshot)
+		api.POST("/files/:dir", apiFilesUpload)
-		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		api.GET("/files/:dir", apiFilesListFiles)
-		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		api.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
 	}
 
 	{
-		root.GET("/snapshots", apiSnapshotsList)
+
-		root.POST("/snapshots", apiSnapshotsCreate)
+		api.GET("/publish", apiPublishList)
-		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.POST("/publish", apiPublishRepoOrSnapshot)
-		root.GET("/snapshots/:name", apiSnapshotsShow)
+		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
-		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
-		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
-		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
 	}
 
 	{
-		root.GET("/packages/:key", apiPackagesShow)
+		api.GET("/snapshots", apiSnapshotsList)
+		api.POST("/snapshots", apiSnapshotsCreate)
+		api.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.GET("/snapshots/:name", apiSnapshotsShow)
+		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
 	}
 
 	{
-		root.GET("/graph.:ext", apiGraph)
+		api.GET("/packages/:key", apiPackagesShow)
+		api.GET("/packages", apiPackages)
+	}
+
+	{
+		api.GET("/graph.:ext", apiGraph)
 	}
 	{
-		root.POST("/db/cleanup", apiDbCleanup)
+		api.POST("/db/cleanup", apiDbCleanup)
 	}
 	{
-		root.GET("/tasks", apiTasksList)
+		api.GET("/tasks", apiTasksList)
-		root.POST("/tasks-clear", apiTasksClear)
+		api.POST("/tasks-clear", apiTasksClear)
-		root.GET("/tasks-wait", apiTasksWait)
+		api.GET("/tasks-wait", apiTasksWait)
-		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
-		root.GET("/tasks/:id/output", apiTasksOutputShow)
+		api.GET("/tasks/:id/output", apiTasksOutputShow)
-		root.GET("/tasks/:id/detail", apiTasksDetailShow)
+		api.GET("/tasks/:id/detail", apiTasksDetailShow)
-		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
-		root.GET("/tasks/:id", apiTasksShow)
+		api.GET("/tasks/:id", apiTasksShow)
-		root.DELETE("/tasks/:id", apiTasksDelete)
+		api.DELETE("/tasks/:id", apiTasksDelete)
-		root.POST("/tasks-dummy", apiTasksDummy)
+		api.POST("/tasks-dummy", apiTasksDummy)
 	}
 
 	return router

api/s3.go

@@ -0,0 +1,19 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary Get S3 buckets
+// @Description Get list of S3 buckets.
+// @Tags S3
+// @Produce  json
+// @Success 200 {array} string "List of S3 buckets"
+// @Router /api/s3 [get]
+func apiS3List(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().S3PublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/snapshot.go

@@ -3,15 +3,23 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/snapshots
+// @Summary Get snapshots
+// @Description Get list of available snapshots. Each snapshot is returned as in “show” API.
+// @Tags Snapshots
+// @Produce  json
+// @Success 200 {array} deb.Snapshot
+// @Router /api/snapshots [get]
 func apiSnapshotsList(c *gin.Context) {
 	SortMethodString := c.Request.URL.Query().Get("sort")
 
@@ -55,14 +63,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
@@ -123,20 +131,20 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
 		err = snapshotCollection.LoadComplete(sources[i])
 		if err != nil {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 
 		resources = append(resources, string(sources[i].ResourceKey()))
 	}
 
-	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		list := deb.NewPackageList()
 
 		// verify package refs and build package list
@@ -160,7 +168,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
-		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
 	})
 }
 
@@ -188,14 +196,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -240,13 +248,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
 	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		_, err := collection.ByName(b.Name)
 		if err == nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
@@ -275,13 +283,13 @@ func apiSnapshotsShow(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -299,13 +307,13 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
@@ -336,32 +344,32 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -385,15 +393,303 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	showPackages(c, snapshot.RefList(), collectionFactory)
 }
+
+type snapshotsMergeParams struct {
+	// List of snapshot names to be merged
+	Sources []string `binding:"required" json:"Sources"     example:"snapshot1"`
+}
+
+// @Summary Snapshot Merge
+// @Description **Merge several source snapshots into a new snapshot**
+// @Description
+// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
+// @Description
+// @Description If only one snapshot is specified, merge copies source into destination.
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot to be created"
+// @Param latest query int false "merge only the latest version of each package"
+// @Param no-remove query int false "all versions of packages are preserved during merge"
+// @Consume json
+// @Param request body snapshotsMergeParams true "Parameters"
+// @Produce  json
+// @Success 200
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/merge [post]
+func apiSnapshotsMerge(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+		body     snapshotsMergeParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if c.Bind(&body) != nil {
+		return
+	}
+
+	if len(body.Sources) < 1 {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
+		return
+	}
+
+	latest := c.Request.URL.Query().Get("latest") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+	overrideMatching := !latest && !noRemove
+
+	if noRemove && latest {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("no-remove and latest are mutually exclusive"))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	sources := make([]*deb.Snapshot, len(body.Sources))
+	resources := make([]string, len(sources))
+	for i := range body.Sources {
+		sources[i], err = snapshotCollection.ByName(body.Sources[i])
+		if err != nil {
+			AbortWithJSONError(c, http.StatusNotFound, err)
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			AbortWithJSONError(c, http.StatusInternalServerError, err)
+			return
+		}
+		resources[i] = string(sources[i].ResourceKey())
+	}
+
+	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		result := sources[0].RefList()
+		for i := 1; i < len(sources); i++ {
+			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+		}
+
+		if latest {
+			result.FilterLatestRefs()
+		}
+
+		sourceDescription := make([]string, len(sources))
+		for i, s := range sources {
+			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
+		}
+
+		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+type snapshotsPullParams struct {
+	// Source name to be searched for packages and dependencies
+	Source string `binding:"required"      json:"Source"            example:"source-snapshot"`
+	// Name of the snapshot to be created
+	Destination string `binding:"required" json:"Destination"       example:"idestination-snapshot"`
+	// List of package queries (i.e. name of package to be pulled from `Source`)
+	Queries []string `binding:"required"   json:"Queries"           example:"xserver-xorg"`
+	// List of architectures (optional)
+	Architectures []string `               json:"Architectures"     example:"amd64, armhf"`
+}
+
+// @Summary Snapshot Pull
+// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
+// @Description
+// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
+// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
+// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
+// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
+// @Description
+// @Description With flag `no-remove` both package versions would stay in the snapshot.
+// @Description
+// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot to be created"
+// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
+// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
+// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
+// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
+// @Consume json
+// @Param request body snapshotsPullParams true "Parameters"
+// @Produce json
+// @Success 200
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/pull [post]
+func apiSnapshotsPull(c *gin.Context) {
+	var (
+		err                 error
+		destinationSnapshot *deb.Snapshot
+		body                snapshotsPullParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if err = c.BindJSON(&body); err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, err)
+		return
+	}
+
+	allMatches := c.Request.URL.Query().Get("all-matches") == "1"
+	dryRun := c.Request.URL.Query().Get("dry-run") == "1"
+	noDeps := c.Request.URL.Query().Get("no-deps") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name> snapshot
+	toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	// Load <Source> snapshot
+	sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
+	}
+
+	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// convert snapshots to package list
+		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		toPackageList.PrepareIndex()
+		sourcePackageList.PrepareIndex()
+
+		var architecturesList []string
+
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = toPackageList.Architectures(false)
+		}
+
+		architecturesList = append(architecturesList, body.Architectures...)
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+		var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+		for _, arch := range architecturesList {
+			archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+		}
+
+		queries := make([]deb.PackageQuery, len(body.Queries))
+		for i, q := range body.Queries {
+			queries[i], err = query.Parse(q)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// Add architecture filter
+			queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+		}
+
+		// Filter with dependencies as requested
+		destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		destinationPackageList.PrepareIndex()
+
+		removedPackages := []string{}
+		addedPackages := []string{}
+		alreadySeen := map[string]bool{}
+
+		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+			key := pkg.Architecture + "_" + pkg.Name
+			_, seen := alreadySeen[key]
+
+			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+			if !noRemove && !seen {
+				// Remove all packages with the same name and architecture
+				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+				for _, p := range packageSearchResults {
+					toPackageList.Remove(p)
+					removedPackages = append(removedPackages, p.String())
+				}
+			}
+
+			// If !allMatches, add only first matching name-arch package
+			if !seen || allMatches {
+				toPackageList.Add(pkg)
+				addedPackages = append(addedPackages, pkg.String())
+			}
+
+			alreadySeen[key] = true
+
+			return nil
+		})
+		alreadySeen = nil
+
+		if dryRun {
+			response := struct {
+				AddedPackages   []string `json:"added_packages"`
+				RemovedPackages []string `json:"removed_packages"`
+			}{
+				AddedPackages:   addedPackages,
+				RemovedPackages: removedPackages,
+			}
+
+			return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
+		}
+
+		// Create <destination> snapshot
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
+	})
+}

api/storage.go

@@ -0,0 +1,43 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

api/task.go

@@ -1,7 +1,6 @@
 package api
 
 import (
-	"fmt"
 	"net/http"
 	"strconv"
 
@@ -10,7 +9,12 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /tasks
+// @Summary Get tasks
+// @Description Get list of available tasks. Each task is returned as in “show” API.
+// @Tags Tasks
+// @Produce  json
+// @Success 200 {array} task.Task
+// @Router /api/tasks [get]
 func apiTasksList(c *gin.Context) {
 	list := context.TaskList()
 	c.JSON(200, list.GetTasks())
@@ -35,13 +39,13 @@ func apiTasksWaitForTaskByID(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	task, err := list.WaitForTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -53,14 +57,14 @@ func apiTasksShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var task task.Task
 	task, err = list.GetTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -72,14 +76,14 @@ func apiTasksOutputShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var output string
 	output, err = list.GetTaskOutputByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -91,14 +95,14 @@ func apiTasksDetailShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var detail interface{}
 	detail, err = list.GetTaskDetailByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -110,13 +114,13 @@ func apiTasksReturnValueShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err := list.GetTaskReturnValueByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -128,14 +132,14 @@ func apiTasksDelete(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var delTask task.Task
 	delTask, err = list.DeleteTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -145,7 +149,7 @@ func apiTasksDelete(c *gin.Context) {
 // POST /tasks-dummy
 func apiTasksDummy(c *gin.Context) {
 	resources := []string{"dummy"}
-	taskName := fmt.Sprintf("Dummy task")
+	taskName := "Dummy task"
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 		out.Printf("Dummy task started\n")
 		detail.Store([]int{1, 2, 3})

api/task_test.go

@@ -71,5 +71,5 @@ func (s *TaskSuite) TestTasksClear(c *C) {
 	c.Check(response.Code, Equals, 200)
 	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "null")
+	c.Check(response.Body.String(), Equals, "[]")
 }

aptly-api.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository API
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/api/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
-
-[Install]
-WantedBy=multi-user.target

aptly.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository server
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/commands/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
-
-[Install]
-WantedBy=multi-user.target

aptly/const.go

@@ -0,0 +1,3 @@
+package aptly
+
+var DistributionFocal = "focal"

aptly/interfaces.go

@@ -35,8 +35,8 @@ type PackagePool interface {
 	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
 	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
 	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
-	// Stat returns Unix stat(2) info
+	// Size returns the size of the given file in bytes.
-	Stat(path string) (os.FileInfo, error)
+	Size(path string) (size int64, err error)
 	// Open returns ReadSeekerCloser to access the file
 	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
@@ -47,6 +47,8 @@ type PackagePool interface {
 
 // LocalPackagePool is implemented by PackagePools residing on the same filesystem
 type LocalPackagePool interface {
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
 	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
 	GenerateTempPath(filename string) (string, error)
 	// Link generates hardlink to destination path
@@ -70,7 +72,7 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file

azure/azure.go

@@ -1,2 +1,129 @@
-// Package azure handles publishing to Azure Storage
 package azure
+
+// Package azure handles publishing to Azure Storage
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+func isBlobNotFound(err error) bool {
+	storageError, ok := err.(azblob.StorageError)
+	return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
+}
+
+type azContext struct {
+	container azblob.ContainerURL
+	prefix    string
+}
+
+func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
+	if err != nil {
+		return nil, err
+	}
+
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
+	result := &azContext{
+		container: containerURL,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+func (az *azContext) blobPath(path string) string {
+	return filepath.Join(az.prefix, path)
+}
+
+func (az *azContext) blobURL(path string) azblob.BlobURL {
+	return az.container.NewBlobURL(az.blobPath(path))
+}
+
+func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(az.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := az.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
+		}
+
+		marker = listBlob.NextMarker
+
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
+		}
+
+		if progress != nil {
+			time.Sleep(time.Duration(500) * time.Millisecond)
+			progress.AddBar(1)
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
+	}
+
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
+		}
+	}
+
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob.ToBlockBlobURL(),
+		uploadOptions,
+	)
+
+	return err
+}
+
+// String
+func (az *azContext) String() string {
+	return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
+}

azure/package_pool.go

@@ -0,0 +1,218 @@
+package azure
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+type PackagePool struct {
+	az *azContext
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates published storage from Azure storage credentials
+func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagePool{az: azctx}, nil
+}
+
+// String
+func (pool *PackagePool) String() string {
+	return pool.az.String()
+}
+
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
+	hash := checksums.SHA256
+	// Use the same path as the file pool, for compat reasons.
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
+}
+
+func (pool *PackagePool) ensureChecksums(
+	poolPath string,
+	checksumStorage aptly.ChecksumStorage,
+) (*utils.ChecksumInfo, error) {
+	targetChecksums, err := checksumStorage.Get(poolPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		blob := pool.az.blobURL(poolPath)
+		download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+		if err != nil {
+			if isBlobNotFound(err) {
+				return nil, nil
+			}
+
+			return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
+		}
+
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
+		if err != nil {
+			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return targetChecksums, nil
+}
+
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	if progress != nil {
+		progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
+		defer progress.ShutdownBar()
+	}
+
+	paths, _, err := pool.az.internalFilelist("", progress)
+	return paths, err
+}
+
+func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
+	return "", errors.New("Azure package pool does not support legacy paths")
+}
+
+func (pool *PackagePool) Size(path string) (int64, error) {
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	return props.ContentLength(), nil
+}
+
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	blob := pool.az.blobURL(path)
+
+	temp, err := os.CreateTemp("", "blob-download")
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating temporary file for blob download")
+	}
+
+	defer os.Remove(temp.Name())
+
+	err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "error downloading blob at %s", path)
+	}
+
+	return temp, nil
+}
+
+func (pool *PackagePool) Remove(path string) (int64, error) {
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
+	}
+
+	_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
+	}
+
+	return props.ContentLength(), nil
+}
+
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		var err error
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	path := pool.buildPoolPath(basename, checksums)
+	blob := pool.az.blobURL(path)
+	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
+	if err != nil {
+		return "", err
+	} else if targetChecksums != nil {
+		// target already exists
+		*checksums = *targetChecksums
+		return path, nil
+	}
+
+	source, err := os.Open(srcPath)
+	if err != nil {
+		return "", err
+	}
+	defer source.Close()
+
+	err = pool.az.putFile(blob, source, checksums.MD5)
+	if err != nil {
+		return "", err
+	}
+
+	if !checksums.Complete() {
+		// need full checksums here
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	err = checksumStorage.Update(path, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	if poolPath == "" {
+		if checksums.SHA256 != "" {
+			poolPath = pool.buildPoolPath(basename, checksums)
+		} else {
+			// No checksums or pool path, so no idea what file to look for.
+			return "", false, nil
+		}
+	}
+
+	size, err := pool.Size(poolPath)
+	if err != nil {
+		return "", false, err
+	} else if size != checksums.Size {
+		return "", false, nil
+	}
+
+	targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
+	if err != nil {
+		return "", false, err
+	} else if targetChecksums == nil {
+		return "", false, nil
+	}
+
+	if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+		checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+		// wrong file?
+		return "", false, nil
+	}
+
+	// fill back checksums
+	*checksums = *targetChecksums
+	return poolPath, true, nil
+}

azure/package_pool_test.go

@@ -0,0 +1,255 @@
+package azure
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type PackagePoolSuite struct {
+	accountName, accountKey, endpoint string
+	pool, prefixedPool                *PackagePool
+	debFile                           string
+	cs                                aptly.ChecksumStorage
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+	cnt := s.pool.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+	c.Assert(err, IsNil)
+
+	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = files.NewMockChecksumStorage()
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
+	})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	var checksum utils.ChecksumInfo
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	// import as different name
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	// double import, should be ok
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with path-relevant checksums already filled in
+	checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
+	path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	checksum := utils.ChecksumInfo{}
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with checksums missing (that aren't needed to find the path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
+		SHA256: "abc",
+		Size:   checksum.Size,
+	}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksums (that aren't needed to find the path)
+	// and no info in checksum storage
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	checksum = utils.ChecksumInfo{Size: 13455}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestSize(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Size("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}

azure/public.go

@@ -2,12 +2,8 @@ package azure
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
-	"io"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"time"
@@ -22,7 +18,8 @@ import (
 type PublishedStorage struct {
 	container azblob.ContainerURL
 	prefix    string
-	pathCache map[string]string
+	az        *azContext
+	pathCache map[string]map[string]string
 }
 
 // Check interface
@@ -30,60 +27,23 @@ var (
 	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
 )
 
-func isEmulatorEndpoint(endpoint string) bool {
-	if h, _, err := net.SplitHostPort(endpoint); err == nil {
-		endpoint = h
-	}
-	if endpoint == "localhost" {
-		return true
-	}
-	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
-	// In this case, eliminate the '[' and ']' in the URL.
-	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
-	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
-		endpoint = endpoint[1 : len(endpoint)-1]
-	}
-	return net.ParseIP(endpoint) != nil
-}
-
 // NewPublishedStorage creates published storage from Azure storage credentials
 func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
-	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	if endpoint == "" {
+	return &PublishedStorage{az: azctx}, nil
-		endpoint = "blob.core.windows.net"
-	}
-
-	var url *url.URL
-	if isEmulatorEndpoint(endpoint) {
-		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
-	} else {
-		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
-
-	result := &PublishedStorage{
-		container: containerURL,
-		prefix:    prefix,
-	}
-
-	return result, nil
 }
 
 // String
 func (storage *PublishedStorage) String() string {
-	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+	return storage.az.String()
 }
 
 // MkDir creates directory recursively under public path
-func (storage *PublishedStorage) MkDir(path string) error {
+func (storage *PublishedStorage) MkDir(_ string) error {
 	// no op for Azure
 	return nil
 }
@@ -106,7 +66,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer source.Close()
 
-	err = storage.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -114,45 +74,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	return err
 }
 
-// putFile uploads file-like object to
-func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
-	path = filepath.Join(storage.prefix, path)
-
-	blob := storage.container.NewBlockBlobURL(path)
-
-	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize: 4 * 1024 * 1024,
-		MaxBuffers: 8,
-	}
-	if len(sourceMD5) > 0 {
-		decodedMD5, err := hex.DecodeString(sourceMD5)
-		if err != nil {
-			return err
-		}
-		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
-			ContentMD5: decodedMD5,
-		}
-	}
-
-	_, err := azblob.UploadStreamToBlockBlob(
-		context.Background(),
-		source,
-		blob,
-		uploadOptions,
-	)
-
-	return err
-}
-
 // RemoveDirs removes directory structure under public path
-func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
+		blob := storage.az.blobURL(filepath.Join(path, filename))
 		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
@@ -164,7 +94,7 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
@@ -174,31 +104,39 @@ func (storage *PublishedStorage) Remove(path string) error {
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, fileName)
+	relFilePath := filepath.Join(publishedRelPath, fileName)
-	poolPath := filepath.Join(storage.prefix, relPath)
+	// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	// FIXME: check how to integrate publishedPrefix:
+	poolPath := storage.az.blobPath(fileName)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist("")
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
 		if err != nil {
 			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
 
-		storage.pathCache = make(map[string]string, len(paths))
+		pathCache = make(map[string]string, len(paths))
 
 		for i := range paths {
-			storage.pathCache[paths[i]] = md5s[i]
+			pathCache[paths[i]] = md5s[i]
 		}
+		storage.pathCache[publishedPrefix] = pathCache
 	}
 
-	destinationMD5, exists := storage.pathCache[relPath]
+	destinationMD5, exists := pathCache[relFilePath]
 	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
@@ -221,9 +159,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	}
 	defer source.Close()
 
-	err = storage.putFile(relPath, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
 	if err == nil {
-		storage.pathCache[relPath] = sourceMD5
+		pathCache[relFilePath] = sourceMD5
 	} else {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}
@@ -231,43 +169,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	return err
 }
 
-func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
-	const delimiter = "/"
-	paths = make([]string, 0, 1024)
-	md5s = make([]string, 0, 1024)
-	prefix = filepath.Join(storage.prefix, prefix)
-	if prefix != "" {
-		prefix += delimiter
-	}
-
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		listBlob, err := storage.container.ListBlobsFlatSegment(
-			context.Background(), marker, azblob.ListBlobsSegmentOptions{
-				Prefix:     prefix,
-				MaxResults: 1000,
-				Details:    azblob.BlobListingDetails{Metadata: true}})
-		if err != nil {
-			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
-		}
-
-		marker = listBlob.NextMarker
-
-		for _, blob := range listBlob.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, blob.Name)
-			} else {
-				paths = append(paths, blob.Name[len(prefix):])
-			}
-			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
-		}
-	}
-
-	return paths, md5s, nil
-}
-
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
-	paths, _, err := storage.internalFilelist(prefix)
+	paths, _, err := storage.az.internalFilelist(prefix, nil)
 	return paths, err
 }
 
@@ -275,8 +179,8 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
 	const leaseDuration = 30
 
-	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
+	dstBlobURL := storage.az.blobURL(dst)
-	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
+	srcBlobURL := storage.az.blobURL(src)
 	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
 	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
 		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
@@ -347,11 +251,10 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		storageError, ok := err.(azblob.StorageError)
+		if isBlobNotFound(err) {
-		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
 			return false, nil
 		}
 		return false, err
@@ -364,7 +267,7 @@ func (storage *PublishedStorage) FileExists(path string) (bool, error) {
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return "", err

azure/public_test.go

@@ -43,14 +43,14 @@ func randString(n int) string {
 func (s *PublishedStorageSuite) SetUpSuite(c *C) {
 	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
 	if s.accountName == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
 	}
 	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 	if s.accountKey == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
@@ -66,7 +66,7 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-	cnt := s.storage.container
+	cnt := s.storage.az.container
 	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
@@ -75,13 +75,13 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-	cnt := s.storage.container
+	cnt := s.storage.az.container
 	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-	blob := s.storage.container.NewBlobURL(path)
+	blob := s.storage.az.container.NewBlobURL(path)
 	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, IsNil)
 	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
@@ -91,7 +91,7 @@ func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-	_, err := s.storage.container.NewBlobURL(path).GetProperties(
+	_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
 		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, NotNil)
 	storageError, ok := err.(azblob.StorageError)
@@ -104,7 +104,7 @@ func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	_, err := azblob.UploadBufferToBlockBlob(
 		context.Background(),
 		data,
-		s.storage.container.NewBlockBlobURL(path),
+		s.storage.az.container.NewBlockBlobURL(path),
 		azblob.UploadToBlockBlobOptions{
 			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
 				ContentMD5: hash[:],
@@ -129,7 +129,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
 }
 
 func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
@@ -300,45 +300,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 
 	// first link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
 
 	// for prefixed storage:
 	// first link from pool
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	// 2nd link from pool, providing wrong path for source file
 	//
 	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with nested file name
-	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))

cmd/api_serve.go

@@ -1,11 +1,15 @@
 package cmd
 
 import (
+	stdcontext "context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/aptly-dev/aptly/api"
 	"github.com/aptly-dev/aptly/systemd/activation"
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}
@@ -55,6 +59,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
+	server := http.Server{Handler: api.Router(context)}
+
+	sigchan := make(chan os.Signal, 1)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go (func() {
+		if _, ok := <-sigchan; ok {
+			server.Shutdown(stdcontext.Background())
+		}
+	})()
+	defer close(sigchan)
+
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
@@ -67,19 +82,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		}
 		defer listener.Close()
 
-		err = http.Serve(listener, api.Router(context))
+		err = server.Serve(listener)
-		if err != nil {
+	} else {
-			return fmt.Errorf("unable to serve: %s", err)
+		server.Addr = listen
-		}
+		err = server.ListenAndServe()
-		return nil
 	}
 
-	err = http.ListenAndServe(listen, api.Router(context))
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
-	if err != nil {
 		return fmt.Errorf("unable to serve: %s", err)
 	}
 
-	return err
+	return nil
 }
 
 func makeCmdAPIServe() *commander.Command {

cmd/cmd.go

@@ -118,7 +118,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
 	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {

cmd/config_show.go

@@ -7,7 +7,7 @@ import (
 	"github.com/smira/commander"
 )
 
-func aptlyConfigShow(cmd *commander.Command, args []string) error {
+func aptlyConfigShow(_ *commander.Command, _ []string) error {
 
 	config := context.Config()
 	prettyJSON, err := json.MarshalIndent(config, "", "    ")

cmd/graph.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -36,7 +35,7 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	buf := bytes.NewBufferString(graph.String())
 
-	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	tempfile, err := os.CreateTemp("", "aptly-graph")
 	if err != nil {
 		return err
 	}

cmd/mirror.go

@@ -9,18 +9,18 @@ import (
 )
 
 func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
-	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
-		return nil, nil
-	}
-
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -20,6 +20,10 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -59,7 +63,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}

cmd/mirror_edit.go

@@ -28,6 +28,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 	}
 
 	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
@@ -43,6 +44,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		case "archive-url":
 			repo.SetArchiveRoot(flag.Value.String())
 			fetchMirror = true
+		case "ignore-signatures":
+			ignoreSignatures = true
 		}
 	})
 
@@ -69,7 +72,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 		}
 
-		err = repo.Fetch(context.Downloader(), verifier)
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}

cmd/mirror_list.go

@@ -24,7 +24,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return aptlyMirrorListTxt(cmd, args)
 }
 
-func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -65,7 +65,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())

cmd/mirror_show.go

@@ -27,7 +27,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	return aptlyMirrorShowTxt(cmd, args)
 }
 
-func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -93,7 +93,7 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/mirror_update.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 
@@ -41,20 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -161,7 +166,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 					var e error
 
 					// provision download location
-					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+						task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+					} else {
+						var file *os.File
+						file, e = os.CreateTemp("", task.File.Filename)
+						if e == nil {
+							task.TempDownPath = file.Name()
+							file.Close()
+						}
+					}
 					if e != nil {
 						pushError(e)
 						continue
@@ -173,7 +187,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
-						ignoreMismatch)
+						ignoreChecksums)
 					if e != nil {
 						pushError(e)
 						continue
@@ -197,6 +211,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	defer func() {
+		for _, task := range queue {
+			if task.TempDownPath == "" {
+				continue
+			}
+
+			if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+				fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+			}
+		}
+	}()
+
 	// Import downloaded files
 	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 

cmd/publish_list.go

@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return aptlyPublishListTxt(cmd, args)
 }
 
-func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -34,7 +34,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
 	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
 		if e != nil {
 			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
 				repo.StoragePrefix(), repo.Distribution, repo.Components())
@@ -77,7 +77,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())

cmd/publish_repo.go

@@ -48,8 +48,10 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_show.go

@@ -24,7 +24,7 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	return aptlyPublishShowTxt(cmd, args)
 }
 
-func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]
@@ -76,7 +76,7 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]

cmd/publish_snapshot.go

@@ -116,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	origin := context.Flags().Lookup("origin").Value.String()
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -132,6 +133,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 	published.Label = context.Flags().Lookup("label").Value.String()
 	published.Suite = context.Flags().Lookup("suite").Value.String()
+	published.Codename = context.Flags().Lookup("codename").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
 
@@ -161,8 +163,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
-			"the same package pool.\n")
 	}
 
 	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
@@ -239,8 +240,10 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/deb"
-	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -64,10 +63,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	}
 
 	for i, component := range components {
-		if !utils.StrSliceHasItem(publishedComponents, component) {
-			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
-		}
-
 		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
@@ -100,6 +95,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -161,6 +160,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -64,6 +64,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
 	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
@@ -119,6 +123,7 @@ Example:
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/repo_include.go

@@ -29,7 +29,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
-	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
 	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
 	collectionFactory := context.NewCollectionFactory()

cmd/repo_list.go

@@ -24,7 +24,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return aptlyRepoListTxt(cmd, args)
 }
 
-func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -71,7 +71,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())

cmd/repo_show.go

@@ -25,7 +25,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	return aptlyRepoShowTxt(cmd, args)
 }
 
-func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -58,7 +58,7 @@ func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/serve.go

@@ -29,7 +29,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}

cmd/snapshot_list.go

@@ -23,7 +23,7 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	return aptlySnapshotListTxt(cmd, args)
 }
 
-func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -59,7 +59,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
 	var err error
 
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)

cmd/snapshot_show.go

@@ -25,7 +25,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	return aptlySnapshotShowTxt(cmd, args)
 }
 
-func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 	var err error
 	name := args[0]
 	collectionFactory := context.NewCollectionFactory()
@@ -85,7 +85,7 @@ func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/task_run.go

@@ -62,11 +62,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 			text, _ = reader.ReadString('\n')
 			if text == "\n" {
 				break
-			} else {
-				text = strings.TrimSpace(text) + ","
-				parsedArgs, _ := shellwords.Parse(text)
-				cmdArgs = append(cmdArgs, parsedArgs...)
 			}
+			text = strings.TrimSpace(text) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
 		}
 
 		if len(cmdArgs) == 0 {

codecov.yml

@@ -3,5 +3,5 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 2%
         if_ci_failed: error

completion.d/_aptly

@@ -457,6 +457,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-distribution=[distribution name to publish]:distribution:($dists)"
                             "-label=[label to publish]:label: "
                             "-suite=[suite to publish]:suite: "
+                            "-codename=[codename to publish]:codename: "
                             "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
                             "-origin=[origin name to publish]:origin: "
                             ${components_options[@]}

completion.d/aptly

@@ -1,5 +1,3 @@
-#!/bin/bash
-
 # (The MIT License)
 #
 # Copyright (c) 2014 Andrey Smirnov
@@ -503,7 +501,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))

console/progress.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/cheggaaa/pb"
+	"github.com/rs/zerolog/log"
 	"github.com/wsxiaoys/terminal/color"
 )
 
@@ -34,6 +35,7 @@ type Progress struct {
 	queue    chan printTask
 	bar      *pb.ProgressBar
 	barShown bool
+	worker   ProgressWorker
 }
 
 // Check interface
@@ -42,16 +44,19 @@ var (
 )
 
 // NewProgress creates new progress instance
-func NewProgress() *Progress {
+func NewProgress(structuredLogging bool) *Progress {
-	return &Progress{
+	p := &Progress{
 		stopped: make(chan bool),
 		queue:   make(chan printTask, 100),
 	}
+
+	p.worker = progressWorkerFactroy(structuredLogging, p)
+	return p
 }
 
 // Start makes progress start its work
 func (p *Progress) Start() {
-	go p.worker()
+	go p.worker.run()
 }
 
 // Shutdown shuts down progress display
@@ -69,7 +74,7 @@ func (p *Progress) Flush() {
 }
 
 // InitBar starts progressbar for count bytes or count items
-func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
+func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}
@@ -173,42 +178,94 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 	}
 }
 
-func (p *Progress) worker() {
+type ProgressWorker interface {
+	run()
+}
+
+func progressWorkerFactroy(structuredLogging bool, progress *Progress) ProgressWorker {
+	if structuredLogging {
+		worker := loggerProgressWorker{progress: progress}
+		return &worker
+	}
+
+	worker := standardProgressWorker{progress: progress}
+	return &worker
+}
+
+type standardProgressWorker struct {
+	progress *Progress
+}
+
+func (w *standardProgressWorker) run() {
 	hasBar := false
 
 	for {
-		task := <-p.queue
+		task := <-w.progress.queue
 		switch task.code {
 		case codeBarEnabled:
 			hasBar = true
 		case codeBarDisabled:
 			hasBar = false
 		case codePrint:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 			fmt.Print(task.message)
 		case codePrintStdErr:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 			fmt.Fprint(os.Stderr, task.message)
 		case codeProgress:
 			if hasBar {
 				fmt.Print("\r" + task.message)
-				p.barShown = true
+				w.progress.barShown = true
 			}
 		case codeHideProgress:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 		case codeFlush:
 			task.reply <- true
 		case codeStop:
-			p.stopped <- true
+			w.progress.stopped <- true
+			return
+		}
+	}
+}
+
+type loggerProgressWorker struct {
+	progress *Progress
+}
+
+func (w *loggerProgressWorker) run() {
+	hasBar := false
+
+	for {
+		task := <-w.progress.queue
+		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
+		case codePrint, codePrintStdErr:
+			log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
+		case codeProgress:
+			if hasBar {
+				log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
+				w.progress.barShown = true
+			}
+		case codeHideProgress:
+			if w.progress.barShown {
+				w.progress.barShown = false
+			}
+		case codeFlush:
+			task.reply <- true
+		case codeStop:
+			w.progress.stopped <- true
 			return
 		}
 	}

console/progress_test.go

@@ -0,0 +1,24 @@
+package console
+
+import (
+	"fmt"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ProgressSuite struct {}
+
+var _ = Suite(&ProgressSuite{})
+
+func (s *ProgressSuite) TestNewProgress(c *C) {
+	p := NewProgress(false)
+	c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &standardProgressWorker{}))
+
+	p = NewProgress(true)
+	c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &loggerProgressWorker{}))
+}

context/context.go

@@ -3,6 +3,7 @@ package context
 
 import (
 	gocontext "context"
+	"errors"
 	"fmt"
 	"math/rand"
 	"os"
@@ -18,6 +19,7 @@ import (
 	"github.com/aptly-dev/aptly/azure"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/etcddb"
 	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/files"
@@ -48,6 +50,7 @@ type AptlyContext struct {
 	publishedStorages map[string]aptly.PublishedStorage
 	dependencyOptions int
 	architecturesList []string
+	structuredLogging bool
 	// Debug features
 	fileCPUProfile *os.File
 	fileMemProfile *os.File
@@ -93,13 +96,15 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 				Fatal(err)
 			}
 		} else {
-			configLocations := []string{
+			homeLocation := filepath.Join(os.Getenv("HOME"), ".aptly.conf")
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+			configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
-				"/etc/aptly.conf",
-			}
 
 			for _, configLocation := range configLocations {
+				// FIXME: check if exists, check if readable
 				err = utils.LoadConfig(configLocation, &utils.Config)
+				if os.IsPermission(err) || os.IsNotExist(err) {
+					continue
+				}
 				if err == nil {
 					break
 				}
@@ -109,7 +114,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
 				// as this is fresh aptly installation, we don't need to support legacy pool locations
 				utils.Config.SkipLegacyPool = true
@@ -195,7 +200,7 @@ func (context *AptlyContext) Progress() aptly.Progress {
 
 func (context *AptlyContext) _progress() aptly.Progress {
 	if context.progress == nil {
-		context.progress = console.NewProgress()
+		context.progress = console.NewProgress(context.structuredLogging)
 		context.progress.Start()
 	}
 
@@ -272,7 +277,7 @@ func (context *AptlyContext) DBPath() string {
 
 // DBPath builds path to database
 func (context *AptlyContext) dbPath() string {
-	return filepath.Join(context.config().RootDir, "db")
+	return filepath.Join(context.config().GetRootDir(), "db")
 }
 
 // Database opens and returns current instance of database
@@ -286,8 +291,18 @@ func (context *AptlyContext) Database() (database.Storage, error) {
 func (context *AptlyContext) _database() (database.Storage, error) {
 	if context.database == nil {
 		var err error
-
+		switch context.config().DatabaseBackend.Type {
-		context.database, err = goleveldb.NewDB(context.dbPath())
+		case "leveldb":
+			if len(context.config().DatabaseBackend.DbPath) == 0 {
+				return nil, errors.New("leveldb databaseBackend config invalid")
+			}
+			dbPath := filepath.Join(context.config().RootDir, context.config().DatabaseBackend.DbPath)
+			context.database, err = goleveldb.NewDB(dbPath)
+		case "etcd":
+			context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
+		default:
+			context.database, err = goleveldb.NewDB(context.dbPath())
+		}
 		if err != nil {
 			return nil, fmt.Errorf("can't instantiate database: %s", err)
 		}
@@ -360,7 +375,26 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	defer context.Unlock()
 
 	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
+		storageConfig := context.config().PackagePoolStorage
+		if storageConfig.Azure != nil {
+			var err error
+			context.packagePool, err = azure.NewPackagePool(
+				storageConfig.Azure.AccountName,
+				storageConfig.Azure.AccountKey,
+				storageConfig.Azure.Container,
+				storageConfig.Azure.Prefix,
+				storageConfig.Azure.Endpoint)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			poolRoot := context.config().PackagePoolStorage.Local.Path
+			if poolRoot == "" {
+				poolRoot = filepath.Join(context.config().RootDir, "pool")
+			}
+
+			context.packagePool = files.NewPackagePool(poolRoot, !context.config().SkipLegacyPool)
+		}
 	}
 
 	return context.packagePool
@@ -374,7 +408,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 	publishedStorage, ok := context.publishedStorages[name]
 	if !ok {
 		if name == "" {
-			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
+			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().GetRootDir(), "public"), "hardlink", "")
 		} else if strings.HasPrefix(name, "filesystem:") {
 			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
@@ -393,7 +427,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
 				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
 				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
-				params.ForceSigV2, params.Debug)
+				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
 			if err != nil {
 				Fatal(err)
 			}
@@ -432,7 +466,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 // UploadPath builds path to upload storage
 func (context *AptlyContext) UploadPath() string {
-	return filepath.Join(context.Config().RootDir, "upload")
+	return filepath.Join(context.Config().GetRootDir(), "upload")
 }
 
 func (context *AptlyContext) pgpProvider() string {
@@ -456,7 +490,7 @@ func (context *AptlyContext) pgpProvider() string {
 	return provider
 }
 
-func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
+func (context *AptlyContext) getGPGFinder() pgp.GPGFinder {
 	switch context.pgpProvider() {
 	case "gpg1":
 		return pgp.GPG1Finder()
@@ -479,7 +513,7 @@ func (context *AptlyContext) GetSigner() pgp.Signer {
 		return &pgp.GoSigner{}
 	}
 
-	return pgp.NewGpgSigner(context.getGPGFinder(provider))
+	return pgp.NewGpgSigner(context.getGPGFinder())
 }
 
 // GetVerifier returns Verifier with respect to provider
@@ -492,7 +526,7 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
 		return &pgp.GoVerifier{}
 	}
 
-	return pgp.NewGpgVerifier(context.getGPGFinder(provider))
+	return pgp.NewGpgVerifier(context.getGPGFinder())
 }
 
 // UpdateFlags sets internal copy of flags in the context
@@ -540,6 +574,11 @@ func (context *AptlyContext) GoContextHandleSignals() {
 	}()
 }
 
+// StructuredLogging allows to set the structuredLogging flag
+func (context *AptlyContext) StructuredLogging(structuredLogging bool) {
+	context.structuredLogging = structuredLogging
+}
+
 // Shutdown shuts context down
 func (context *AptlyContext) Shutdown() {
 	context.Lock()
@@ -561,6 +600,9 @@ func (context *AptlyContext) Shutdown() {
 			context.fileMemProfile = nil
 		}
 	}
+	if context.taskList != nil {
+		context.taskList.Stop()
+	}
 	if context.database != nil {
 		context.database.Close()
 		context.database = nil

database/etcddb/batch.go

@@ -0,0 +1,53 @@
+package etcddb
+
+import (
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type EtcDBatch struct {
+	s   *EtcDStorage
+	ops []clientv3.Op
+}
+
+type WriteOptions struct {
+	NoWriteMerge bool
+	Sync         bool
+}
+
+func (b *EtcDBatch) Put(key []byte, value []byte) (err error) {
+	b.ops = append(b.ops, clientv3.OpPut(string(key), string(value)))
+	return
+}
+
+func (b *EtcDBatch) Delete(key []byte) (err error) {
+	b.ops = append(b.ops, clientv3.OpDelete(string(key)))
+	return
+}
+
+func (b *EtcDBatch) Write() (err error) {
+	kv := clientv3.NewKV(b.s.db)
+
+	batchSize := 128
+	for i := 0; i < len(b.ops); i += batchSize {
+		txn := kv.Txn(Ctx)
+		end := i + batchSize
+		if end > len(b.ops) {
+			end = len(b.ops)
+		}
+
+		batch := b.ops[i:end]
+		txn.Then(batch...)
+		_, err = txn.Commit()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return
+}
+
+// batch should implement database.Batch
+var (
+	_ database.Batch = &EtcDBatch{}
+)

database/etcddb/database.go

@@ -0,0 +1,32 @@
+package etcddb
+
+import (
+	"context"
+	"time"
+
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+var Ctx = context.TODO()
+
+func internalOpen(url string) (cli *clientv3.Client, err error) {
+	cfg := clientv3.Config{
+		Endpoints:            []string{url},
+		DialTimeout:          30 * time.Second,
+		MaxCallSendMsgSize:   2147483647, // (2048 * 1024 * 1024) - 1
+		MaxCallRecvMsgSize:   2147483647,
+		DialKeepAliveTimeout: 7200 * time.Second,
+	}
+
+	cli, err = clientv3.New(cfg)
+	return
+}
+
+func NewDB(url string) (database.Storage, error) {
+	cli, err := internalOpen(url)
+	if err != nil {
+		return nil, err
+	}
+	return &EtcDStorage{url, cli, ""}, nil
+}

database/etcddb/database_test.go

@@ -0,0 +1,158 @@
+package etcddb_test
+
+import (
+	"testing"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/etcddb"
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type EtcDDBSuite struct {
+	url string
+	db  database.Storage
+}
+
+var _ = Suite(&EtcDDBSuite{})
+
+func (s *EtcDDBSuite) SetUpTest(c *C) {
+	var err error
+	s.db, err = etcddb.NewDB("127.0.0.1:2379")
+	c.Assert(err, IsNil)
+}
+
+func (s *EtcDDBSuite) TestSetUpTest(c *C) {
+	var err error
+	s.db, err = etcddb.NewDB("127.0.0.1:2379")
+	c.Assert(err, IsNil)
+}
+
+func (s *EtcDDBSuite) TestGetPut(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+	var err error
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *EtcDDBSuite) TestDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	_, err = s.db.Get(key)
+	c.Assert(err, IsNil)
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+
+}
+
+func (s *EtcDDBSuite) TestByPrefix(c *C) {
+	//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), Equals, database.ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), IsNil)
+
+	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+}
+
+func (s *EtcDDBSuite) TestHasPrefix(c *C) {
+	//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
+func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
+	var (
+		key    = []byte("key")
+		key2   = []byte("key2")
+		value  = []byte("value")
+		value2 = []byte("value2")
+	)
+	transaction, err := s.db.OpenTransaction()
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	c.Assert(err, IsNil)
+	transaction.Put(key2, value2)
+	v, err := s.db.Get(key)
+	c.Check(v, DeepEquals, value)
+        err = transaction.Delete(key)
+	c.Assert(err, IsNil)
+
+	_, err = transaction.Get(key2)
+	c.Assert(err, IsNil)
+
+	v2, err := transaction.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = transaction.Get(key)
+	c.Assert(err, IsNil)
+
+	err = transaction.Commit()
+	c.Check(err, IsNil)
+
+	v2, err = transaction.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = transaction.Get(key)
+	c.Assert(err, NotNil)
+}
+

database/etcddb/storage.go

@@ -0,0 +1,202 @@
+package etcddb
+
+import (
+	"github.com/aptly-dev/aptly/database"
+	"github.com/pborman/uuid"
+	clientv3 "go.etcd.io/etcd/client/v3"
+
+	"fmt"
+)
+
+type EtcDStorage struct {
+	url       string
+	db        *clientv3.Client
+	tmpPrefix string // prefix for temporary DBs
+}
+
+// CreateTemporary creates new DB of the same type in temp dir
+func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
+	tmp := uuid.NewRandom().String()
+	return &EtcDStorage{
+		url:       s.url,
+		db:        s.db,
+		tmpPrefix: tmp,
+	}, nil
+}
+
+func (s *EtcDStorage) applyPrefix(key []byte) []byte {
+	if len(s.tmpPrefix) != 0 {
+		return append([]byte(s.tmpPrefix+"/"), key...)
+	}
+	return key
+}
+
+// Get key value from etcd
+func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
+	realKey := s.applyPrefix(key)
+	getResp, err := s.db.Get(Ctx, string(realKey))
+	if err != nil {
+		return
+	}
+	for _, kv := range getResp.Kvs {
+		value = kv.Value
+		break
+	}
+	if len(value) == 0 {
+		err = database.ErrNotFound
+		return
+	}
+	return
+}
+
+// Put saves key to etcd, if key has the same value in DB already, it is not saved
+func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
+	realKey := s.applyPrefix(key)
+	_, err = s.db.Put(Ctx, string(realKey), string(value))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Delete removes key from etcd
+func (s *EtcDStorage) Delete(key []byte) (err error) {
+	realKey := s.applyPrefix(key)
+	_, err = s.db.Delete(Ctx, string(realKey))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// KeysByPrefix returns all keys that start with prefix
+func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
+	realPrefix := s.applyPrefix(prefix)
+	result := make([][]byte, 0, 20)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return nil
+	}
+	for _, ev := range getResp.Kvs {
+		key := ev.Key
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, key)
+	}
+	return result
+}
+
+// FetchByPrefix returns all values with keys that start with prefix
+func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
+	realPrefix := s.applyPrefix(prefix)
+	result := make([][]byte, 0, 20)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return nil
+	}
+	for _, kv := range getResp.Kvs {
+		valc := make([]byte, len(kv.Value))
+		copy(valc, kv.Value)
+		result = append(result, kv.Value)
+	}
+
+	return result
+}
+
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
+	realPrefix := s.applyPrefix(prefix)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return false
+	}
+	return getResp.Count > 0
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
+	realPrefix := s.applyPrefix(prefix)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return err
+	}
+
+	for _, kv := range getResp.Kvs {
+		err := proc(kv.Key, kv.Value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Close finishes etcd connect
+func (s *EtcDStorage) Close() error {
+	// do not close temporary db
+	if len(s.tmpPrefix) != 0 {
+		return nil
+	}
+	if s.db == nil {
+		return nil
+	}
+	err := s.db.Close()
+	s.db = nil
+	return err
+}
+
+// Reopen tries to open (re-open) the database
+func (s *EtcDStorage) Open() error {
+	if s.db != nil {
+		return nil
+	}
+	var err error
+	s.db, err = internalOpen(s.url)
+	return err
+}
+
+// CreateBatch creates a Batch object
+func (s *EtcDStorage) CreateBatch() database.Batch {
+	if s.db == nil {
+		return nil
+	}
+	return &EtcDBatch{
+		s: s,
+	}
+}
+
+// OpenTransaction creates new transaction.
+func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) {
+	tmpdb, err := s.CreateTemporary()
+	if err != nil {
+		return nil, err
+	}
+	return &transaction{s: s, tmpdb: tmpdb}, nil
+}
+
+// CompactDB does nothing for etcd
+func (s *EtcDStorage) CompactDB() error {
+	return nil
+}
+
+// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys)
+func (s *EtcDStorage) Drop() error {
+	if len(s.tmpPrefix) != 0 {
+		getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix())
+		if err != nil {
+			return nil
+		}
+		for _, kv := range getResp.Kvs {
+			_, err = s.db.Delete(Ctx, string(kv.Key))
+			if err != nil {
+				return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key)
+			}
+		}
+	}
+	return nil
+}
+
+// Check interface
+var (
+	_ database.Storage = &EtcDStorage{}
+)

database/etcddb/transaction.go

@@ -0,0 +1,75 @@
+package etcddb
+
+import (
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type transaction struct {
+	s     *EtcDStorage
+	tmpdb database.Storage
+	ops   []clientv3.Op
+}
+
+// Get implements database.Reader interface.
+func (t *transaction) Get(key []byte) (value []byte, err error) {
+	value, err = t.tmpdb.Get(key)
+	// if not found, search main db
+	if err != nil {
+		value, err = t.s.Get(key)
+	}
+	return
+}
+
+// Put implements database.Writer interface.
+func (t *transaction) Put(key, value []byte) (err error) {
+	err = t.tmpdb.Put(key, value)
+	if err != nil {
+		return
+	}
+	t.ops = append(t.ops, clientv3.OpPut(string(key), string(value)))
+	return
+}
+
+// Delete implements database.Writer interface.
+func (t *transaction) Delete(key []byte) (err error) {
+	err = t.tmpdb.Delete(key)
+	if err != nil {
+		return
+	}
+	t.ops = append(t.ops, clientv3.OpDelete(string(key)))
+	return
+}
+
+func (t *transaction) Commit() (err error) {
+	kv := clientv3.NewKV(t.s.db)
+
+	batchSize := 128
+	for i := 0; i < len(t.ops); i += batchSize {
+		txn := kv.Txn(Ctx)
+		end := i + batchSize
+		if end > len(t.ops) {
+			end = len(t.ops)
+		}
+
+		batch := t.ops[i:end]
+		txn.Then(batch...)
+		_, err = txn.Commit()
+		if err != nil {
+			panic(err)
+		}
+	}
+	t.ops = []clientv3.Op{}
+
+	return
+}
+
+// Discard is safe to call after Commit(), it would be no-op
+func (t *transaction) Discard() {
+	t.ops = []clientv3.Op{}
+	t.tmpdb.Drop()
+	return
+}
+
+// transaction should implement database.Transaction
+var _ database.Transaction = &transaction{}

database/goleveldb/storage.go

@@ -3,7 +3,6 @@ package goleveldb
 import (
 	"bytes"
 	"errors"
-	"io/ioutil"
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -19,7 +18,7 @@ type storage struct {
 
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *storage) CreateTemporary() (database.Storage, error) {
-	tempdir, err := ioutil.TempDir("", "aptly")
+	tempdir, err := os.MkdirTemp("", "aptly")
 	if err != nil {
 		return nil, err
 	}

deb/changes.go

@@ -4,16 +4,17 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/saracen/walker"
 )
 
 // Changes is a result of .changes file parsing
@@ -39,7 +40,7 @@ func NewChanges(path string) (*Changes, error) {
 		ChangesName: filepath.Base(path),
 	}
 
-	c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	c.TempDir, err = os.MkdirTemp(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
 	}
@@ -221,12 +222,12 @@ func (c *Changes) GetField(field string) string {
 }
 
 // MatchesDependency implements PackageLike interface
-func (c *Changes) MatchesDependency(d Dependency) bool {
+func (c *Changes) MatchesDependency(_ Dependency) bool {
 	return false
 }
 
 // MatchesArchitecture implements PackageLike interface
-func (c *Changes) MatchesArchitecture(arch string) bool {
+func (c *Changes) MatchesArchitecture(_ string) bool {
 	return false
 }
 
@@ -248,6 +249,8 @@ func (c *Changes) GetArchitecture() string {
 
 // CollectChangesFiles walks filesystem collecting all .changes files
 func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
+	changesFilesLock := &sync.Mutex{}
+
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -256,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 			continue
 		}
 		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+			err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
-				if err3 != nil {
-					return err3
-				}
 				if info.IsDir() {
 					return nil
 				}
 
 				if strings.HasSuffix(info.Name(), ".changes") {
+					changesFilesLock.Lock()
+					defer changesFilesLock.Unlock()
 					changesFiles = append(changesFiles, path)
 				}
 

deb/changes_test.go

@@ -45,7 +45,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 
 	s.checksumStorage = files.NewMockChecksumStorage()
 	s.packagePool = files.NewPackagePool(s.Dir, false)
-	s.progress = console.NewProgress()
+	s.progress = console.NewProgress(false)
 	s.progress.Start()
 }
 

deb/contents.go

@@ -64,7 +64,7 @@ func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
 		currentPkgs [][]byte
 	)
 
-	err = index.db.ProcessByPrefix(index.prefix, func(key []byte, value []byte) error {
+	err = index.db.ProcessByPrefix(index.prefix, func(key []byte, _ []byte) error {
 		// cut prefix
 		key = key[prefixLen:]
 

deb/format.go

@@ -136,7 +136,7 @@ func isMultilineField(field string, isRelease bool) bool {
 
 // Write single field from Stanza to writer.
 //
-//nolint: interfacer
+// nolint: interfacer
 func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
 	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")

deb/graph.go

@@ -119,8 +119,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
-			"label": fmt.Sprintf("%sPublished %s/%s|comp: %s|arch: %s%s", labelStart,
+			"label": fmt.Sprintf("%sPublished %s|comp: %s|arch: %s%s", labelStart,
-				repo.Prefix, repo.Distribution, strings.Join(repo.Components(), " "),
+				repo.GetPath(), strings.Join(repo.Components(), " "),
 				strings.Join(repo.Architectures, ", "), labelEnd),
 		})
 

deb/import.go

@@ -5,14 +5,19 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/saracen/walker"
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
 func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
+	packageFilesLock := &sync.Mutex{}
+	otherFilesLock := &sync.Mutex{}
+
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -21,18 +26,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			continue
 		}
 		if info.IsDir() {
-			err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
+			err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
-				if err3 != nil {
-					return err3
-				}
 				if info.IsDir() {
 					return nil
 				}
 
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
+					packageFilesLock.Lock()
+					defer packageFilesLock.Unlock()
 					packageFiles = append(packageFiles, path)
 				} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+					otherFilesLock.Lock()
+					defer otherFilesLock.Unlock()
 					otherFiles = append(otherFiles, path)
 				}
 
@@ -71,13 +77,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		list.PrepareIndex()
 	}
 
-	transaction, err := collection.db.OpenTransaction()
+	checksumStorage := checksumStorageProvider(collection.db)
-	if err != nil {
-		return nil, nil, err
-	}
-	defer transaction.Discard()
-
-	checksumStorage := checksumStorageProvider(transaction)
 
 	for _, file := range packageFiles {
 		var (
@@ -201,7 +201,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
-		err = collection.UpdateInTransaction(p, transaction)
+		err = collection.Update(p)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)
 			failedFiles = append(failedFiles, file)
@@ -227,6 +227,6 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	err = transaction.Commit()
+	err = nil // reset error as only failed files are reported
 	return
 }

deb/index_files.go

@@ -143,19 +143,20 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	}
 
 	if signer != nil {
+		gpgExt := ".gpg"
 		if file.detachedSign {
-			err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+			err = signer.DetachedSign(file.tempFilename, file.tempFilename+gpgExt)
 			if err != nil {
 				return fmt.Errorf("unable to detached sign file: %s", err)
 			}
 
 			if file.parent.suffix != "" {
-				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt)] =
-					filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+					filepath.Join(file.parent.basePath, file.relativePath+gpgExt)
 			}
 
-			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt),
-				file.tempFilename+".gpg")
+				file.tempFilename+gpgExt)
 			if err != nil {
 				return fmt.Errorf("unable to publish file: %s", err)
 			}
@@ -248,7 +249,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 	}
 }
 
-func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
+func (files *indexFiles) PackageIndex(component, arch string, udeb bool, installer bool, distribution string) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false
 	}
@@ -263,7 +264,11 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bo
 			if udeb {
 				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
 			} else if installer {
-				relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
+				if distribution == aptly.DistributionFocal {
+					relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "legacy-images", "SHA256SUMS")
+				} else {
+					relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
+				}
 			} else {
 				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
 			}

deb/list.go

@@ -145,7 +145,7 @@ func (l *PackageList) Add(p *Package) error {
 	l.packages[key] = p
 
 	if l.indexed {
-		for _, provides := range p.Provides {
+		for _, provides := range p.ProvidedPackages() {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 
@@ -215,7 +215,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 func (l *PackageList) Remove(p *Package) {
 	delete(l.packages, l.keyFunc(p))
 	if l.indexed {
-		for _, provides := range p.Provides {
+		for _, provides := range p.ProvidedPackages() {
 			for i, pkg := range l.providesIndex[provides] {
 				if pkg.Equals(p) {
 					// remove l.ProvidesIndex[provides][i] w/o preserving order
@@ -351,7 +351,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 						cache[hash] = satisfied
 					}
 
-					if !satisfied && !ok {
+					if !satisfied {
 						variantsMissing = append(variantsMissing, dep)
 					}
 
@@ -366,6 +366,8 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 		}
 	}
 
+	missing = depSliceDeduplicate(missing)
+
 	if progress != nil {
 		progress.ShutdownBar()
 	}
@@ -417,7 +419,7 @@ func (l *PackageList) PrepareIndex() {
 		l.packagesIndex[i] = p
 		i++
 
-		for _, provides := range p.Provides {
+		for _, provides := range p.ProvidedPackages() {
 			l.providesIndex[provides] = append(l.providesIndex[provides], p)
 		}
 	}
@@ -470,21 +472,25 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 			searchResults = append(searchResults, p)
 
 			if !allMatches {
-				break
+				return
 			}
 		}
 
 		i++
 	}
 
-	if dep.Relation == VersionDontCare {
+	providers, ok := l.providesIndex[dep.Pkg]
-		for _, p := range l.providesIndex[dep.Pkg] {
+	if !ok {
-			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+		return
+	}
+	for _, p := range providers {
+		if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+			if p.MatchesDependency(dep) {
 				searchResults = append(searchResults, p)
+			}
 
-				if !allMatches {
+			if !allMatches {
-					break
+				return
-				}
 			}
 		}
 	}

deb/list_test.go

@@ -300,6 +300,14 @@ func (s *PackageListSuite) TestSearch(c *C) {
 	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
 	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
 	c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
+
+	// Provides with version number
+	python3CFFIBackend := &Package{Name: "python3-cffi-backend", Version: "1.15.1-5+b1", Architecture: "amd64", Provides: []string{"python3-cffi-backend-api-9729", "python3-cffi-backend-api-max (= 10495)", "python3-cffi-backend-api-min (= 9729)"}}
+	err := s.il2.Add(python3CFFIBackend)
+	c.Check(err, IsNil)
+	c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionGreaterOrEqual, Version: "9729"}, false), DeepEquals, []*Package{python3CFFIBackend})
+	c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionLess, Version: "9729"}, false), IsNil)
+	c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionDontCare}, false), DeepEquals, []*Package{python3CFFIBackend})
 }
 
 func (s *PackageListSuite) TestFilter(c *C) {

deb/local.go

@@ -116,7 +116,7 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
 		return result
 	}
 
-	collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+	collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding local repo: %s\n", err)
@@ -219,7 +219,7 @@ func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
 
 // ForEach runs method for each repository
 func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
-	return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+	return collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding repo: %s\n", err)

deb/package.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/rs/zerolog/log"
 )
 
 // Package is single instance of Debian package
@@ -189,7 +190,12 @@ func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, componen
 		return nil, err
 	}
 
-	relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	var relPath string
+	if repo.Distribution == aptly.DistributionFocal {
+		relPath = filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "legacy-images")
+	} else {
+		relPath = filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	}
 	for i := range files {
 		files[i].downloadPath = relPath
 
@@ -307,6 +313,23 @@ func (p *Package) GetField(name string) string {
 	}
 }
 
+// ProvidedPackages returns just the package names of the provided packages (without version numbers such as
+// `(= 1.2.3)`).
+func (p *Package) ProvidedPackages() []string {
+	result := make([]string, len(p.Provides))
+	for i, provided := range p.Provides {
+		providedDep, err := ParseDependency(provided)
+		if err != nil {
+			// Should never happen, but I included this, so it definitely has the old behavior in case there is no
+			// special syntax.
+			result[i] = provided
+		} else {
+			result[i] = providedDep.Pkg
+		}
+	}
+	return result
+}
+
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
 	if p.Architecture == ArchitectureAll && arch != ArchitectureSource {
@@ -316,24 +339,75 @@ func (p *Package) MatchesArchitecture(arch string) bool {
 	return p.Architecture == arch
 }
 
+func JoinErrors(errs ...error) error {
+	var combinedErr error
+	for _, err := range errs {
+		if err != nil {
+			if combinedErr == nil {
+				combinedErr = err
+			} else {
+				combinedErr = fmt.Errorf("%w\n%v", combinedErr, err)
+			}
+		}
+	}
+	return combinedErr
+}
+
+// providesDependency checks if the package `Provide:`s the dependency, assuming that the architecture matches.
+// If the `Provides:` entry includes a version number, it will be considered when checking the dependency.
+func (p *Package) providesDependency(dep Dependency) (bool, error) {
+	var errs []error // won't cause an allocation in case of no errors
+	for _, provided := range p.Provides {
+		providedDep, err := ParseDependency(provided)
+		if err != nil {
+			errs = append(errs, err)
+		}
+		// The only relation allowed here is `=`.
+		// > The relations allowed are [...]. The exception is the Provides field, for which only = is allowed.
+		// > [...]
+		// > A Provides field may contain version numbers, and such a version number will be considered when
+		// > considering a dependency on or conflict with the virtual package name.
+		// -- https://www.debian.org/doc/debian-policy/ch-relationships.html
+		switch providedDep.Relation {
+		case VersionDontCare:
+			if providedDep.Pkg == dep.Pkg {
+				return true, nil
+			}
+		case VersionEqual:
+			providedVersion := providedDep.Version
+			if providedDep.Pkg == dep.Pkg && versionSatisfiesDependency(providedVersion, dep) {
+				return true, nil
+			}
+		default:
+			errs = append(errs, fmt.Errorf("unsupported relation in Provides: %s", providedDep.String()))
+		}
+	}
+	return false, JoinErrors(errs...)
+}
+
 // MatchesDependency checks whether package matches specified dependency
 func (p *Package) MatchesDependency(dep Dependency) bool {
 	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
 		return false
 	}
 
-	if dep.Relation == VersionDontCare {
+	providesDep, err := p.providesDependency(dep)
-		if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
+	if err != nil {
-			return true
+		log.Warn().Err(err).Msg("Error while checking if package provides dependency")
-		}
+	}
-		return dep.Pkg == p.Name
+	if providesDep {
+		return true
 	}
 
 	if dep.Pkg != p.Name {
 		return false
 	}
 
-	r := CompareVersions(p.Version, dep.Version)
+	return versionSatisfiesDependency(p.Version, dep)
+}
+
+func versionSatisfiesDependency(version string, dep Dependency) bool {
+	r := CompareVersions(version, dep.Version)
 
 	switch dep.Relation {
 	case VersionEqual:
@@ -347,13 +421,15 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
 	case VersionGreaterOrEqual:
 		return r >= 0
 	case VersionPatternMatch:
-		matched, err := filepath.Match(dep.Version, p.Version)
+		matched, err := filepath.Match(dep.Version, version)
 		return err == nil && matched
 	case VersionRegexp:
-		return dep.Regexp.FindStringIndex(p.Version) != nil
+		return dep.Regexp.FindStringIndex(version) != nil
+	case VersionDontCare:
+		return true
+	default:
+		panic(fmt.Sprintf("unknown relation: %d", dep.Relation))
 	}
-
-	panic("unknown relation")
 }
 
 // GetName returns package name
@@ -621,9 +697,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		publishedDirectory := filepath.Join(prefix, relPath)
+		err = publishedStorage.LinkFromPool(prefix, relPath, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
-
-		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
 		if err != nil {
 			return err
 		}

deb/package_collection.go

@@ -317,7 +317,7 @@ func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList)
 }
 
 // Search is not implemented
-func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
+func (collection *PackageCollection) Search(_ Dependency, _ bool) (searchResults []*Package) {
 	panic("Not implemented")
 }
 

deb/package_test.go

@@ -333,9 +333,12 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 
 	// Provides
 	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
-	p.Provides = []string{"fun", "game"}
+	p.Provides = []string{"fun (= 42)", "game"}
 	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
 	c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionDontCare}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionGreaterOrEqual, Version: "42"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionLess, Version: "42"}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {

deb/publish.go

@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -44,6 +43,7 @@ type PublishedRepo struct {
 	ButAutomaticUpgrades string
 	Label                string
 	Suite                string
+	Codename             string
 	// Architectures is a list of all architectures published
 	Architectures []string
 	// SourceKind is "local"/"repo"
@@ -70,6 +70,9 @@ type PublishedRepo struct {
 
 	// Provide index files per hash also
 	AcquireByHash bool
+
+	// Support multiple distributions
+	MultiDist bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -153,13 +156,14 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 // distribution and architectures are user-defined properties
 // components & sources are lists of component to source mapping (*Snapshot or *LocalRepo)
 func NewPublishedRepo(storage, prefix, distribution string, architectures []string,
-	components []string, sources []interface{}, collectionFactory *CollectionFactory) (*PublishedRepo, error) {
+	components []string, sources []interface{}, collectionFactory *CollectionFactory, multiDist bool) (*PublishedRepo, error) {
 	result := &PublishedRepo{
 		UUID:          uuid.New(),
 		Storage:       storage,
 		Architectures: architectures,
 		Sources:       make(map[string]string),
 		sourceItems:   make(map[string]repoSourceItem),
+		MultiDist:     multiDist,
 	}
 
 	if len(sources) == 0 {
@@ -197,9 +201,6 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		if distribution == "" || component == "" {
 			rootDistributions, rootComponents := walkUpTree(source, collectionFactory)
 			if distribution == "" {
-				for i := range rootDistributions {
-					rootDistributions[i] = strings.Replace(rootDistributions[i], "/", "-", -1)
-				}
 				discoveredDistributions = append(discoveredDistributions, rootDistributions...)
 			}
 			if component == "" {
@@ -264,10 +265,6 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		}
 	}
 
-	if strings.Contains(distribution, "/") {
-		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
-	}
-
 	result.Distribution = distribution
 
 	// only fields which are unique by all given snapshots are set on published
@@ -284,7 +281,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 	return result, nil
 }
 
-// MarshalJSON requires object to be "loaded completely"
+// MarshalJSON requires object to filled by "LoadShallow" or "LoadComplete"
 func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 	type sourceInfo struct {
 		Component, Name string
@@ -312,6 +309,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"Label":                p.Label,
 		"Origin":               p.Origin,
 		"Suite":                p.Suite,
+		"Codename":             p.Codename,
 		"NotAutomatic":         p.NotAutomatic,
 		"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
 		"Prefix":               p.Prefix,
@@ -321,6 +319,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"Storage":              p.Storage,
 		"SkipContents":         p.SkipContents,
 		"AcquireByHash":        p.AcquireByHash,
+		"MultiDist":            p.MultiDist,
 	})
 }
 
@@ -366,6 +365,10 @@ func (p *PublishedRepo) String() string {
 		extras = append(extras, fmt.Sprintf("suite: %s", p.Suite))
 	}
 
+	if p.Codename != "" {
+		extras = append(extras, fmt.Sprintf("codename: %s", p.Codename))
+	}
+
 	extra = strings.Join(extras, ", ")
 
 	if extra != "" {
@@ -418,6 +421,29 @@ func (p *PublishedRepo) Components() []string {
 	return result
 }
 
+// Components returns sorted list of published repo source names
+func (p *PublishedRepo) SourceNames() []string {
+	var sources = []string{}
+
+	for _, component := range p.Components() {
+		var source string
+
+		item := p.sourceItems[component]
+		if item.snapshot != nil {
+			source = item.snapshot.Name
+		} else if item.localRepo != nil {
+			source = item.localRepo.Name
+		} else {
+			panic("no snapshot/localRepo")
+		}
+
+		sources = append(sources, fmt.Sprintf("%s:%s", source, component))
+	}
+
+	sort.Strings(sources)
+	return sources
+}
+
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
 	if p.SourceKind != SourceLocalRepo {
@@ -437,7 +463,10 @@ func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) {
 		panic("not snapshot publish")
 	}
 
-	item := p.sourceItems[component]
+	item, exists := p.sourceItems[component]
+	if !exists {
+		item = repoSourceItem{}
+	}
 	item.snapshot = snapshot
 	p.sourceItems[component] = item
 
@@ -513,6 +542,14 @@ func (p *PublishedRepo) GetSuite() string {
 	return p.Suite
 }
 
+// GetCodename returns default or manual Codename:
+func (p *PublishedRepo) GetCodename() string {
+	if p.Codename == "" {
+		return p.Distribution
+	}
+	return p.Codename
+}
+
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
@@ -582,7 +619,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 
 	var tempDir string
-	tempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
+	tempDir, err = os.MkdirTemp(os.TempDir(), "aptly")
 	if err != nil {
 		return err
 	}
@@ -605,7 +642,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
-			indexes.PackageIndex(component, arch, false, false)
+			indexes.PackageIndex(component, arch, false, false, p.Distribution)
 		}
 
 		list.PrepareIndex()
@@ -627,9 +664,18 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						if err2 != nil {
 							return err2
 						}
-						relPath = filepath.Join("pool", component, poolDir)
+						if p.MultiDist {
+							relPath = filepath.Join("pool", p.Distribution, component, poolDir)
+						} else {
+							relPath = filepath.Join("pool", component, poolDir)
+						}
+
 					} else {
-						relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
+						if p.Distribution == aptly.DistributionFocal {
+							relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "legacy-images")
+						} else {
+							relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
+						}
 					}
 
 					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite)
@@ -667,7 +713,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						}
 					}
 
-					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller, p.Distribution).BufWriter()
 					if err != nil {
 						return err
 					}
@@ -721,7 +767,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			// For all architectures, pregenerate .udeb indexes
 			for _, arch := range p.Architectures {
-				indexes.PackageIndex(component, arch, true, false)
+				indexes.PackageIndex(component, arch, true, false, p.Distribution)
 			}
 		}
 
@@ -735,6 +781,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
 				release["Suite"] = p.GetSuite()
+				release["Codename"] = p.GetCodename()
 				if p.AcquireByHash {
 					release["Acquire-By-Hash"] = "yes"
 				}
@@ -793,7 +840,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.GetSuite()
-	release["Codename"] = p.Distribution
+	release["Codename"] = p.GetCodename()
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	if p.AcquireByHash {
@@ -953,8 +1000,11 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
 	return batch.Write()
 }
 
-// LoadComplete loads additional information for remote repo
+// LoadShallow loads basic information on the repo's sources
-func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
+//
+// This does not *fully* load in the sources themselves and their packages.
+// It's useful if you just want to use JSON serialization without loading in unnecessary things.
+func (collection *PublishedRepoCollection) LoadShallow(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
 	repo.sourceItems = make(map[string]repoSourceItem)
 
 	if repo.SourceKind == SourceSnapshot {
@@ -965,10 +1015,6 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 			if err != nil {
 				return
 			}
-			err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot)
-			if err != nil {
-				return
-			}
 
 			repo.sourceItems[component] = item
 		}
@@ -980,6 +1026,30 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 			if err != nil {
 				return
 			}
+
+			item.packageRefs = &PackageRefList{}
+			repo.sourceItems[component] = item
+		}
+	} else {
+		panic("unknown SourceKind")
+	}
+
+	return
+}
+
+// LoadComplete loads complete information on the sources of the repo *and* their packages
+func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
+	collection.LoadShallow(repo, collectionFactory)
+
+	if repo.SourceKind == SourceSnapshot {
+		for _, item := range repo.sourceItems {
+			err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot)
+			if err != nil {
+				return
+			}
+		}
+	} else if repo.SourceKind == SourceLocalRepo {
+		for component, item := range repo.sourceItems {
 			err = collectionFactory.LocalRepoCollection().LoadComplete(item.localRepo)
 			if err != nil {
 				return
@@ -998,13 +1068,10 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 				}
 			}
 
-			item.packageRefs = &PackageRefList{}
 			err = item.packageRefs.Decode(encoded)
 			if err != nil {
 				return
 			}
-
-			repo.sourceItems[component] = item
 		}
 	} else {
 		panic("unknown SourceKind")
@@ -1086,7 +1153,7 @@ func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*Publi
 
 // ForEach runs method for each repository
 func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	return collection.db.ProcessByPrefix([]byte("U"), func(key, blob []byte) error {
+	return collection.db.ProcessByPrefix([]byte("U"), func(_, blob []byte) error {
 		r := &PublishedRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding published repo: %s\n", err)
@@ -1104,18 +1171,10 @@ func (collection *PublishedRepoCollection) Len() int {
 	return len(collection.list)
 }
 
-// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
+func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix string, components []string,
-func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
+	collectionFactory *CollectionFactory, progress aptly.Progress) (map[string][]string, error) {
-	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
-
-	collection.loadList()
-
-	var err error
 	referencedFiles := map[string][]string{}
-
+	processedComponentRefs := map[string]*PackageRefList{}
-	if progress != nil {
-		progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", "))
-	}
 
 	for _, r := range collection.list {
 		if r.Prefix == prefix {
@@ -1134,16 +1193,28 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 				continue
 			}
 
-			err = collection.LoadComplete(r, collectionFactory)
+			if err := collection.LoadComplete(r, collectionFactory); err != nil {
-			if err != nil {
+				return nil, err
-				return err
 			}
 
 			for _, component := range components {
 				if utils.StrSliceHasItem(repoComponents, component) {
-					packageList, err := NewPackageListFromRefList(r.RefList(component), collectionFactory.PackageCollection(), progress)
+					unseenRefs := r.RefList(component)
+					processedRefs := processedComponentRefs[component]
+					if processedRefs != nil {
+						unseenRefs = unseenRefs.Subtract(processedRefs)
+					} else {
+						processedRefs = NewPackageRefList()
+					}
+
+					if unseenRefs.Len() == 0 {
+						continue
+					}
+					processedComponentRefs[component] = processedRefs.Merge(unseenRefs, false, true)
+
+					packageList, err := NewPackageListFromRefList(unseenRefs, collectionFactory.PackageCollection(), progress)
 					if err != nil {
-						return err
+						return nil, err
 					}
 
 					packageList.ForEach(func(p *Package) error {
@@ -1163,6 +1234,24 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 		}
 	}
 
+	return referencedFiles, nil
+}
+
+// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
+func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
+	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+
+	collection.loadList()
+
+	if progress != nil {
+		progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", "))
+	}
+
+	referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress)
+	if err != nil {
+		return err
+	}
+
 	for _, component := range components {
 		sort.Strings(referencedFiles[component])
 

deb/publish_bench_test.go

@@ -0,0 +1,113 @@
+package deb
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+)
+
+func BenchmarkListReferencedFiles(b *testing.B) {
+	const defaultComponent = "main"
+	const repoCount = 16
+	const repoPackagesCount = 1024
+	const uniqPackagesCount = 64
+
+	tmpDir, err := os.MkdirTemp("", "aptly-bench")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	db, err := goleveldb.NewOpenDB(tmpDir)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer db.Close()
+
+	factory := NewCollectionFactory(db)
+	packageCollection := factory.PackageCollection()
+	repoCollection := factory.LocalRepoCollection()
+	publishCollection := factory.PublishedRepoCollection()
+
+	sharedRefs := NewPackageRefList()
+	{
+		transaction, err := db.OpenTransaction()
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		for pkgIndex := 0; pkgIndex < repoPackagesCount-uniqPackagesCount; pkgIndex++ {
+			p := &Package{
+				Name:         fmt.Sprintf("pkg-shared_%d", pkgIndex),
+				Version:      "1",
+				Architecture: "amd64",
+			}
+			p.UpdateFiles(PackageFiles{PackageFile{
+				Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex),
+			}})
+
+			packageCollection.UpdateInTransaction(p, transaction)
+			sharedRefs.Refs = append(sharedRefs.Refs, p.Key(""))
+		}
+
+		sort.Sort(sharedRefs)
+
+		if err := transaction.Commit(); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	for repoIndex := 0; repoIndex < repoCount; repoIndex++ {
+		refs := NewPackageRefList()
+
+		transaction, err := db.OpenTransaction()
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		for pkgIndex := 0; pkgIndex < uniqPackagesCount; pkgIndex++ {
+			p := &Package{
+				Name:         fmt.Sprintf("pkg%d_%d", repoIndex, pkgIndex),
+				Version:      "1",
+				Architecture: "amd64",
+			}
+			p.UpdateFiles(PackageFiles{PackageFile{
+				Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex),
+			}})
+
+			packageCollection.UpdateInTransaction(p, transaction)
+			refs.Refs = append(refs.Refs, p.Key(""))
+		}
+
+		if err := transaction.Commit(); err != nil {
+			b.Fatal(err)
+		}
+
+		sort.Sort(refs)
+
+		repo := NewLocalRepo(fmt.Sprintf("repo%d", repoIndex), "comment")
+		repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex)
+		repo.DefaultComponent = defaultComponent
+		repo.UpdateRefList(refs.Merge(sharedRefs, false, true))
+		repoCollection.Add(repo)
+
+		publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false)
+		if err != nil {
+			b.Fatal(err)
+		}
+		publishCollection.Add(publish)
+	}
+
+	db.CompactDB()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := publishCollection.listReferencedFilesByComponent("test", []string{defaultComponent}, factory, nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

deb/publish_test.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
@@ -133,19 +134,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p2)
 	s.packageCollection.Update(s.p3)
 
-	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
 	s.repo.SkipContents = true
 
-	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
 	s.repo2.SkipContents = true
 
-	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	s.repo3.SkipContents = true
 
-	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
 	s.repo4.SkipContents = true
 
-	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
 	s.repo5.SkipContents = true
 }
 
@@ -177,19 +178,71 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
 	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
 	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
 
-	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
+	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources")
 	c.Check(func() {
-		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	}, PanicMatches, "sources and components should be equal in size")
 	c.Check(func() {
-		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
+		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false)
 	}, PanicMatches, "interface conversion:.*")
 
-	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 
-	_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
-	c.Check(err, ErrorMatches, "invalid distribution wheezy/updates, '/' is not allowed")
+	c.Check(err, IsNil)
+}
+
+func (s *PublishedRepoSuite) TestMultiDistPool(c *C) {
+	repo, err := NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, true)
+	c.Assert(err, IsNil)
+	err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
+	c.Assert(err, IsNil)
+
+	publishedStorage := files.NewPublishedStorage(s.root, "", "")
+
+	c.Check(repo.Architectures, DeepEquals, []string{"i386"})
+
+	rf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Origin"], Equals, "ppa squeeze")
+	c.Check(st["Components"], Equals, "main")
+	c.Check(st["Architectures"], Equals, "i386")
+
+	pf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(pf, false, false)
+
+	for i := 0; i < 3; i++ {
+		st, err = cfr.ReadStanza()
+		c.Assert(err, IsNil)
+
+		c.Check(st["Filename"], Equals, "pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	}
+
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+	c.Assert(st, IsNil)
+
+	drf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
+	c.Assert(err, IsNil)
+
+	cfr = NewControlFileReader(drf, true, false)
+	st, err = cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(st["Archive"], Equals, "squeeze")
+	c.Check(st["Architecture"], Equals, "i386")
+
+	_, err = os.Stat(filepath.Join(publishedStorage.PublicPath(), "ppa/pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
+	c.Assert(err, IsNil)
+
 }
 
 func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
@@ -248,7 +301,7 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 			errorExpected: "invalid prefix .*",
 		},
 	} {
-		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+		repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
 		if t.errorExpected != "" {
 			c.Check(err, ErrorMatches, t.errorExpected)
 		} else {
@@ -258,51 +311,51 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
-	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "wheezy")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"non-free"})
 
-	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
 	s.localRepo.DefaultComponent = "contrib"
 	s.factory.LocalRepoCollection().Update(s.localRepo)
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "precise")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
 
 	s.localRepo.DefaultDistribution = "precise/updates"
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, IsNil)
-	c.Check(repo.Distribution, Equals, "precise-updates")
+	c.Check(repo.Distribution, Equals, "precise/updates")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
 
-	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
@@ -390,10 +443,10 @@ func (s *PublishedRepoSuite) TestString(c *C) {
 		"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	c.Check(s.repo2.String(), Equals,
 		"ppa/maverick [] publishes {main: [local1]: comment1}")
-	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
 	c.Check(repo.String(), Equals,
 		"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
-	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
 	c.Check(repo.String(), Equals,
 		"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
 	repo.Origin = "myorigin"
@@ -450,13 +503,22 @@ type PublishedRepoCollectionSuite struct {
 var _ = Suite(&PublishedRepoCollectionSuite{})
 
 func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
+	s.SetUpPackages()
+
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
 
-	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
+	snap1Refs := NewPackageRefList()
-	s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
+	snap1Refs.Refs = [][]byte{s.p1.Key(""), s.p2.Key("")}
+	sort.Sort(snap1Refs)
+	s.snap1 = NewSnapshotFromRefList("snap1", []*Snapshot{}, snap1Refs, "desc1")
+
+	snap2Refs := NewPackageRefList()
+	snap2Refs.Refs = [][]byte{s.p3.Key("")}
+	sort.Sort(snap2Refs)
+	s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2")
 
 	s.snapshotCollection.Add(s.snap1)
 	s.snapshotCollection.Add(s.snap2)
@@ -464,11 +526,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 	s.localRepo = NewLocalRepo("local1", "comment1")
 	s.factory.LocalRepoCollection().Add(s.localRepo)
 
-	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
-	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false)
-	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory, false)
-	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
-	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
 
 	s.collection = s.factory.PublishedRepoCollection()
 }
@@ -534,7 +596,7 @@ func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	c.Assert(r.sourceItems["main"].snapshot, IsNil)
 	c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
 	c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
-	c.Assert(r.RefList("main").Len(), Equals, 0)
+	c.Assert(r.RefList("main").Len(), Equals, 2)
 
 	r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
 	c.Assert(err, IsNil)
@@ -625,6 +687,51 @@ func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
 	c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
 }
 
+func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
+	c.Check(s.factory.PackageCollection().Update(s.p1), IsNil)
+	c.Check(s.factory.PackageCollection().Update(s.p2), IsNil)
+	c.Check(s.factory.PackageCollection().Update(s.p3), IsNil)
+
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
+	c.Check(s.collection.Add(s.repo4), IsNil)
+	c.Check(s.collection.Add(s.repo5), IsNil)
+
+	files, err := s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
+	c.Assert(err, IsNil)
+	for _, v := range files {
+		sort.Strings(v)
+	}
+	c.Check(files, DeepEquals, map[string][]string{
+		"contrib": {
+			"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
+			"a/alien-arena/mars-invaders_7.40-2_i386.deb",
+		},
+		"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
+	})
+
+	snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
+	s.snapshotCollection.Add(snap3)
+
+	// Ensure that adding a second publish point with matching files doesn't give duplicate results.
+	repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
+	c.Check(err, IsNil)
+	c.Check(s.collection.Add(repo3), IsNil)
+
+	files, err = s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
+	c.Assert(err, IsNil)
+	for _, v := range files {
+		sort.Strings(v)
+	}
+	c.Check(files, DeepEquals, map[string][]string{
+		"contrib": {
+			"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
+			"a/alien-arena/mars-invaders_7.40-2_i386.deb",
+		},
+		"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
+	})
+}
+
 type PublishedRepoRemoveSuite struct {
 	PackageListMixinSuite
 	db                                  database.Storage
@@ -650,11 +757,11 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 
 	s.snapshotCollection.Add(s.snap1)
 
-	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
-	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
-	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
+	s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
-	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
-	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
+	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
 
 	s.collection = s.factory.PublishedRepoCollection()
 	s.collection.Add(s.repo1)

deb/query.go

@@ -138,7 +138,7 @@ func (q *NotQuery) Matches(pkg PackageLike) bool {
 }
 
 // Fast is false
-func (q *NotQuery) Fast(list PackageCatalog) bool {
+func (q *NotQuery) Fast(_ PackageCatalog) bool {
 	return false
 }
 
@@ -197,7 +197,7 @@ func (q *FieldQuery) Query(list PackageCatalog) (result *PackageList) {
 }
 
 // Fast depends on the query
-func (q *FieldQuery) Fast(list PackageCatalog) bool {
+func (q *FieldQuery) Fast(_ PackageCatalog) bool {
 	return false
 }
 
@@ -265,7 +265,7 @@ func (q *PkgQuery) Matches(pkg PackageLike) bool {
 }
 
 // Fast is always true for package query
-func (q *PkgQuery) Fast(list PackageCatalog) bool {
+func (q *PkgQuery) Fast(_ PackageCatalog) bool {
 	return true
 }
 
@@ -280,12 +280,12 @@ func (q *PkgQuery) String() string {
 }
 
 // Matches on specific properties
-func (q *MatchAllQuery) Matches(pkg PackageLike) bool {
+func (q *MatchAllQuery) Matches(_ PackageLike) bool {
 	return true
 }
 
 // Fast is always true for match all query
-func (q *MatchAllQuery) Fast(list PackageCatalog) bool {
+func (q *MatchAllQuery) Fast(_ PackageCatalog) bool {
 	return true
 }
 

deb/reflist.go

@@ -71,7 +71,9 @@ func (l *PackageRefList) Encode() []byte {
 
 // Decode decodes msgpack representation into PackageRefLit
 func (l *PackageRefList) Decode(input []byte) error {
-	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+	handle := &codec.MsgpackHandle{}
+	handle.ZeroCopy = true
+	decoder := codec.NewDecoderBytes(input, handle)
 	return decoder.Decode(l)
 }
 
@@ -194,31 +196,21 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 
 	// until we reached end of both lists
 	for il < ll || ir < lr {
-		// if we've exhausted left list, pull the rest from the right
+		var rl, rr []byte
-		if il == ll {
+		if il < ll {
-			pr, err = packageCollection.ByKey(r.Refs[ir])
+			rl = l.Refs[il]
-			if err != nil {
-				return nil, err
-			}
-			result = append(result, PackageDiff{Left: nil, Right: pr})
-			ir++
-			continue
 		}
-		// if we've exhausted right list, pull the rest from the left
+		if ir < lr {
-		if ir == lr {
+			rr = r.Refs[ir]
-			pl, err = packageCollection.ByKey(l.Refs[il])
-			if err != nil {
-				return nil, err
-			}
-			result = append(result, PackageDiff{Left: pl, Right: nil})
-			il++
-			continue
 		}
 
-		// refs on both sides are present, load them
-		rl, rr := l.Refs[il], r.Refs[ir]
 		// compare refs
 		rel := bytes.Compare(rl, rr)
+		// an unset ref is less than all others, but since it represents the end
+		// of a reflist, it should be *greater*, so flip the comparison result
+		if rl == nil || rr == nil {
+			rel = -rel
+		}
 
 		if rel == 0 {
 			// refs are identical, so are packages, advance pointer
@@ -227,14 +219,14 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
 			pl, pr = nil, nil
 		} else {
 			// load pl & pr if they haven't been loaded before
-			if pl == nil {
+			if pl == nil && rl != nil {
 				pl, err = packageCollection.ByKey(rl)
 				if err != nil {
 					return nil, err
 				}
 			}
 
-			if pr == nil {
+			if pr == nil && rr != nil {
 				pr, err = packageCollection.ByKey(rr)
 				if err != nil {
 					return nil, err
@@ -310,38 +302,41 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConfli
 			overridenName = nil
 			overriddenArch = nil
 		} else {
-			partsL := bytes.Split(rl, []byte(" "))
+			if !ignoreConflicting || overrideMatching {
-			archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
+				partsL := bytes.Split(rl, []byte(" "))
+				archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
 
-			partsR := bytes.Split(rr, []byte(" "))
+				partsR := bytes.Split(rr, []byte(" "))
-			archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
+				archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
 
-			if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
+				if !ignoreConflicting && bytes.Equal(archL, archR) &&
-				// conflicting duplicates with same arch, name, version, but different file hash
+					bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
-				result.Refs = append(result.Refs, r.Refs[ir])
+					// conflicting duplicates with same arch, name, version, but different file hash
-				il++
-				ir++
-				overridenName = nil
-				overriddenArch = nil
-				continue
-			}
-
-			if overrideMatching {
-				if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
-					// this package has already been overridden on the right
-					il++
-					continue
-				}
-
-				if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
-					// override with package from the right
 					result.Refs = append(result.Refs, r.Refs[ir])
 					il++
 					ir++
-					overriddenArch = archL
+					overridenName = nil
-					overridenName = nameL
+					overriddenArch = nil
 					continue
 				}
+
+				if overrideMatching {
+					if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
+						// this package has already been overridden on the right
+						il++
+						continue
+					}
+
+					if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
+						// override with package from the right
+						result.Refs = append(result.Refs, r.Refs[ir])
+						il++
+						ir++
+						overriddenArch = archL
+						overridenName = nameL
+						continue
+					}
+				}
 			}
 
 			// otherwise append smallest of two

deb/reflist_bench_test.go

@@ -0,0 +1,47 @@
+package deb
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+)
+
+func BenchmarkReflistSimpleMerge(b *testing.B) {
+	const count = 4096
+
+	l := NewPackageRefList()
+	r := NewPackageRefList()
+
+	for i := 0; i < count; i++ {
+		if i%2 == 0 {
+			l.Refs = append(l.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
+		} else {
+			r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
+		}
+	}
+
+	sort.Sort(l)
+	sort.Sort(r)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		l.Merge(r, false, true)
+	}
+}
+
+func BenchmarkReflistDecode(b *testing.B) {
+	const count = 4096
+
+	r := NewPackageRefList()
+	for i := 0; i < count; i++ {
+		r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
+	}
+
+	sort.Sort(r)
+	data := r.Encode()
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		(&PackageRefList{}).Decode(data)
+	}
+}

deb/reflist_test.go

@@ -237,6 +237,41 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 
 }
 
+func (s *PackageRefListSuite) TestDiffCompactsAtEnd(c *C) {
+	db, _ := goleveldb.NewOpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+
+	packages := []*Package{
+		{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //0
+		{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //1
+		{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //2
+	}
+
+	for _, p := range packages {
+		coll.Update(p)
+	}
+
+	listA := NewPackageList()
+	listA.Add(packages[0])
+
+	listB := NewPackageList()
+	listB.Add(packages[1])
+	listB.Add(packages[2])
+
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
+
+	diffAB, err := reflistA.Diff(reflistB, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAB, HasLen, 2)
+
+	c.Check(diffAB[0].Left, IsNil)
+	c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
+
+	c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
+}
+
 func (s *PackageRefListSuite) TestMerge(c *C) {
 	db, _ := goleveldb.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)

deb/remote.go

@@ -259,6 +259,9 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
 // InstallerPath returns path of Packages files for given component and
 // architecture
 func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
+	if repo.Distribution == aptly.DistributionFocal {
+		return fmt.Sprintf("%s/installer-%s/current/legacy-images/SHA256SUMS", component, architecture)
+	}
 	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
 }
 
@@ -270,17 +273,29 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 }
 
 // Fetch updates information about repository
-func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
+func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier, ignoreSignatures bool) error {
 	var (
 		release, inrelease, releasesig *os.File
 		err                            error
 	)
 
-	if verifier == nil {
+	if ignoreSignatures {
 		// 0. Just download release file to temporary URL
 		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
-			return err
+			// 0.1 try downloading InRelease, ignore and strip signature
+			inrelease, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("InRelease").String())
+			if err != nil {
+				return err
+			}
+			if verifier == nil {
+				return fmt.Errorf("no verifier specified")
+			}
+			release, err = verifier.ExtractClearsigned(inrelease)
+			if err != nil {
+				return err
+			}
+			goto ok
 		}
 	} else {
 		// 1. try InRelease file
@@ -428,8 +443,7 @@ ok:
 }
 
 // DownloadPackageIndexes downloads & parses package index files
-func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, _ *CollectionFactory, ignoreSignatures bool, ignoreChecksums bool) error {
-	ignoreMismatch bool) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
 	}
@@ -462,14 +476,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 	for _, info := range packagesPaths {
 		path, kind, component, architecture := info[0], info[1], info[2], info[3]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreChecksums)
 
 		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
 			if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
 				// checking if gpg file is only needed when checksums matches are required.
 				// otherwise there actually has been no candidate found and we can continue
-				if ignoreMismatch {
+				if ignoreChecksums {
 					continue
 				}
 
@@ -486,7 +500,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					return err
 				}
 
-				if verifier != nil {
+				if verifier != nil && !ignoreSignatures {
 					hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
 					var filesig *os.File
 					filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
@@ -777,7 +791,7 @@ func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, un
 		return result
 	}
 
-	collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+	collection.db.ProcessByPrefix([]byte("R"), func(_, blob []byte) error {
 		r := &RemoteRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding remote repo: %s\n", err)
@@ -880,7 +894,7 @@ func (collection *RemoteRepoCollection) ByUUID(uuid string) (*RemoteRepo, error)
 
 // ForEach runs method for each repository
 func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error) error {
-	return collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+	return collection.db.ProcessByPrefix([]byte("R"), func(_, blob []byte) error {
 		r := &RemoteRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding mirror: %s\n", err)