ci: add release notes

and update Releasing.md

.dockerignore

@@ -0,0 +1,10 @@
+.go/
+.git/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+unit.out
+aptly.test
+build/
+dpkgs/

.flake8

@@ -1,7 +1,7 @@
 [flake8]
-max-line-length = 200
+max-line-length = 240
 ignore = E126,E241,E741,W504
 include =
     system
 exclude =
-    system/env
+    system/env

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/workflows/ci.yml

@@ -1,5 +1,3 @@
-# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
-
 name: CI
 
 on:
@@ -13,32 +11,17 @@ on:
 defaults:
   run:
     # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
-    shell: bash --noprofile --norc -eo pipefail -x {0}
+    shell: bash --noprofile --norc -eo pipefail {0}
 
 env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
-  build:
-    name: test
-    runs-on: ubuntu-20.04
-    continue-on-error: ${{ matrix.allow_failure }}
+  test:
+    name: "Test (Ubuntu 22.04)"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
     timeout-minutes: 30
-    strategy:
-      fail-fast: true
-      matrix:
-        go: [1.16, 1.17, 1.18]
-        run_long_tests: [no]
-        allow_failure: [false]
-        include:
-          # Disable this for now as it's not clear how to select the latest master branch
-          # version of go using actions/setup-go@v2.
-          # - go: master
-          #   run_long_tests: no
-          #   allow_failure: true
-          - go: 1.17
-            run_long_tests: yes
-            allow_failure: false
 
     env:
       NO_FTP_ACCESS: yes
@@ -47,101 +30,275 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go }}
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: v1.45.0
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install O/S packages
+      - name: "Install Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
 
-      - name: Install Python packages
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Run flake8"
         run: |
-          pip install six packaging appdirs virtualenv
-          pip install -U pip setuptools
-          pip install -r system/requirements.txt
+          make flake8
 
-      - name: Install Azurite
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Install Azurite"
         id: azuright
         uses: potatoqualitee/azuright@v1.1
         with:
           directory: ${{ runner.temp }}
 
-      - name: Get aptly version
-        run: |
-           make version
-
-      - name: Make
+      - name: "Run Unit Tests"
         env:
-          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
-          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
           AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          make
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make test
 
-      - name: Upload code coverage
-        if: matrix.run_long_tests
+      - name: "Run Benchmark"
+        run: |
+          COVERAGE_DIR=${{ runner.temp }} make bench
+
+      - name: "Run System Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make system-test
+
+      - name: "Merge Code Coverage"
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+
+      - name: "Upload Code Coverage"
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.txt
 
-  release:
-    name: release
-    needs: build
-    runs-on: ubuntu-20.04
+  ci-debian-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        arch: ["amd64", "i386" , "arm64" , "armhf"]
+        include:
+          - name: "Debian 13/testing"
+            suite: trixie
+            image: debian:trixie-slim
+          - name: "Debian 12/bookworm"
+            suite: bookworm
+            image: debian:bookworm-slim
+          - name: "Debian 11/bullseye"
+            suite: bullseye
+            image: debian:bullseye-slim
+          - name: "Debian 10/buster"
+            suite: buster
+            image: debian:buster-slim
+          - name: "Ubuntu 24.04"
+            suite: noble
+            image: ubuntu:24.04
+          - name: "Ubuntu 22.04"
+            suite: jammy
+            image: ubuntu:22.04
+          - name: "Ubuntu 20.04"
+            suite: focal
+            image: ubuntu:20.04
+    container:
+      image: ${{ matrix.image }}
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - name: "Install packages"
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
+                          binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf \
+                          libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
+                          gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
         with:
-          # fetch the whole repot for `git describe` to
-          # work and get the nightly verion
+          # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: Setup Go
-        uses: actions/setup-go@v2
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
 
-      - name: Make Release
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Build Debian packages"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make dpkg DEBARCH=${{ matrix.arch }}
+
+      - name: "Check aptly credentials"
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          found=no
+          if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
+            found=yes
+          fi
+          echo "Aptly credentials available: $found"
+          echo "FOUND=$found" >> $GITHUB_OUTPUT
+        id: aptlycreds
+
+      - name: "Publish CI release to aptly"
+        if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
+
+      - name: "Publish release to aptly"
+        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
+
+  ci-binary-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, freebsd, darwin]
+        goarch: ["386", "amd64", "arm", "arm64"]
+        exclude:
+          - goos: darwin
+            goarch: 386
+          - goos: darwin
+            goarch: arm
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read Go Version"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
         env:
           GOBIN: /usr/local/bin
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
         run: |
-          make release
+          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
 
-      - name: Publish nightly release to aptly
-        if: github.ref == 'refs/heads/master'
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh nightly
-
-      - name: Publish release to aptly
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
         if: startsWith(github.event.ref, 'refs/tags')
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh release
-
-      - name: Upload artifacts to GitHub Release
-        if: startsWith(github.event.ref, 'refs/tags')
-        uses: softprops/action-gh-release@v1
         with:
-          body: Release ${{ github.ref }} generated by the CI.
-          files: build/*
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+
+  gh-release:
+    name: "Github Release"
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: ci-binary-build
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Download Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: out/
+
+      - name: "Create Release Notes"
+        run: |
+          echo -e "## Changes\n\n" > out/release-notes.md
+          dpkg-parsechangelog -S Changes | tail -n +4 >> out/release-notes.md
+
+      - name: "Release"
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "Aptly Release ${{ steps.releaseversion.outputs.VERSION }}"
+          files: "out/**/aptly_*.zip"
+          body_path: "out/release-notes.md"

.github/workflows/golangci-lint.yml

@@ -0,0 +1,72 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+
+      - name: Install and initialize swagger
+        run: |
+          go install github.com/swaggo/swag/cmd/swag@latest
+          swag init -q --markdownFiles docs
+        shell: sh
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.54.1
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/scripts/upload-artifacts.sh

@@ -0,0 +1,170 @@
+#!/bin/sh
+
+set -e
+
+builds=build/
+packages=${builds}*.deb
+folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
+aptly_user="$APTLY_USER"
+aptly_password="$APTLY_PASSWORD"
+aptly_api="https://aptly-ops.aptly.info"
+version=`make version`
+
+action=$1
+dist=$2
+
+usage() {
+    echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
+    echo "       $0 release" >&2
+}
+
+# repos and publish must be created beforehand:
+#!/bin/sh
+#for dist in buster bullseye bookworm focal jammy noble
+#do
+#        for build in ci release
+#        do
+#            echo
+#            echo "# Creating and publishing $build/$dist"
+#            aptly repo create -distribution=$dist -component=main aptly-$build-$dist
+#            aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
+#                               -distribution=$dist -batch -keyring=aptly.pub \
+#                               aptly-$build-$dist \
+#                               s3:repo.aptly.info:$build
+#       done
+#done
+
+if [ -z "$action" ]; then
+    usage
+    exit 1
+fi
+
+if [ "action" = "ci" ] && [ -z "$dist" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
+    usage
+    echo Error: please set APTLY_USER and APTLY_PASSWORD
+    exit 1
+fi
+
+echo "Publishing version '$version' to $action for $dist...\n"
+
+upload()
+{
+    echo "\nUploading files:"
+    for file in $packages; do
+        echo " - $file"
+        jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+    done
+}
+
+cleanup() {
+    echo "\nCleanup..."
+    jsonret=`curl -fsS -X DELETE  -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+}
+trap cleanup EXIT
+
+sleeptime=5
+retries=60
+wait_task()
+{
+    _id=$1
+    _success=0
+    sleep $sleeptime
+    for t in `seq $retries`
+    do
+        jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
+        _state=`echo $jsonret | jq .State`
+        if [ "$_state" = "2" ]; then
+            _success=1
+            curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id
+            break
+        fi
+        if [ "$_state" = "3" ]; then
+            echo Error: task failed
+            return 1
+        fi
+        sleep $sleeptime
+    done
+    if [ "$_success" -ne 1 ]; then
+        echo Error: task timeout
+        return 1
+    fi
+    return 0
+}
+
+add_packages() {
+    _aptly_repository=$1
+    _folder=$2
+    jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$_aptly_repository/file/$_folder?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: adding packages to $_aptly_repository failed"
+        exit 1
+    fi
+}
+
+update_publish() {
+    _publish=$1
+    _dist=$2
+    jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+        '{"AcquireByHash": true, "MultiDist": true,
+          "Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
+        -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: publish failed"
+        exit 1
+    fi
+}
+
+if [ "$action" = "ci" ]; then
+    if echo "$version" | grep -vq "+"; then
+       # skip ci when on release tag
+       exit 0
+    fi
+
+    aptly_repository=aptly-ci-$dist
+    aptly_published=s3:repo.aptly.info:ci
+
+elif [ "$action" = "release" ]; then
+    aptly_repository=aptly-release-$dist
+    aptly_published=s3:repo.aptly.info:release
+fi
+
+upload
+
+echo "\nAdding packages to $aptly_repository ..."
+add_packages $aptly_repository $folder
+
+echo "\nUpdating published repo $aptly_published ..."
+update_publish $aptly_published $dist
+
+# if [ "$action" = "OBSOLETErelease" ]; then
+#     aptly_repository=aptly-release
+#     aptly_snapshot=aptly-$version
+#     aptly_published=s3:repo.aptly.info:./squeeze
+#
+#     echo "\nAdding packages to $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
+#     echo
+#
+#     echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
+#         "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
+#     echo
+#
+#     echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
+#     curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+#         "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
+#                                    \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
+#                                                  \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
+#         -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
+#     echo
+# fi
+

.gitignore

@@ -2,11 +2,14 @@
 *.o
 *.a
 *.so
+unit.out
 
 # Folders
 _obj
 _test
 
+tmp/
+
 # Architecture specific extensions/prefixes
 *.[568vq]
 [568vq].out
@@ -32,13 +35,42 @@ root/
 man/aptly.1.html
 man/aptly.1.ronn
 
-.goxc.local.json
-
 system/env/
 
 # created by make build for release artifacts
+VERSION
+aptly.test
+
 build/
 
-pgp/keyrings/aptly2*.gpg
-pgp/keyrings/aptly2*.gpg~
-pgp/keyrings/.#*
+system/files/aptly2.gpg~
+system/files/aptly2_passphrase.gpg~
+
+*.creds
+
+.go/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+
+# debian
+debian/.debhelper/
+debian/aptly.substvars
+debian/aptly/
+debian/debhelper-build-stamp
+debian/files
+debian/aptly-api/
+debian/*.debhelper
+debian/*.debhelper.log
+debian/aptly-api.substvars
+debian/aptly-dbg.substvars
+debian/aptly-dbg/
+usr/bin/aptly
+dpkgs/
+debian/changelog.dpkg-bak
+
+docs/docs.go
+docs/swagger.json
+docs/swagger.yaml
+docs/swagger.conf

.golangci.yml

@@ -5,7 +5,6 @@ run:
 linters:
   disable-all: true
   enable:
-   - deadcode
    - goconst
    - gofmt
    - goimports
@@ -14,6 +13,4 @@ linters:
    - misspell
    - revive
    - staticcheck
-   - structcheck
-   - varcheck
    - vetshadow

.goxc.json

@@ -1,45 +0,0 @@
-{
-	"AppName": "aptly",
-	"ArtifactsDest": "xc-out/",
-	"TasksExclude": [
-		"rmbin",
-		"go-test",
-		"go-vet"
-	],
-    "TasksAppend": [
-		"bintray"
-    ],
-    "TaskSettings": {
-        "debs": {
-            "metadata": {
-                "maintainer": "Andrey Smirnov",
-                "maintainer-email": "me@smira.ru",
-                "description": "Debian repository management tool"
-            },
-            "metadata-deb": {
-                "License": "MIT",
-                "Homepage": "https://www.aptly.info/",
-                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-				"Suggests": "graphviz"
-            },
-            "other-mapped-files": {
-                "/": "root/"
-            }
-        },
-        "bintray": {
-            "repository": "aptly",
-            "subject": "smira",
-            "package": "aptly",
-            "downloadspage": "bintray.md"
-        }
-    },
-	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
-	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
-	"MainDirsExclude": "_man,vendor",
-	"BuildSettings": {
-		"LdFlagsXVars": {
-			"Version": "main.Version"
-		}
-	},
-	"ConfigVersion": "0.9"
-}

.travis.yml

@@ -1,106 +0,0 @@
-dist: xenial
-sudo: required
-
-language: go
-
-go_import_path: github.com/aptly-dev/aptly
-
-addons:
-  apt:
-    packages:
-    - python-virtualenv
-    - graphviz
-    - gnupg2
-    - gpgv2
-
-env:
-  global:
-    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
-    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
-    - NO_FTP_ACCESS: "yes"
-    - BOTO_CONFIG: /dev/null
-    - GO111MODULE: "on"
-    - GOPROXY: https://proxy.golang.org
-
-matrix:
-  allow_failures:
-    - go: master
-      env: RUN_LONG_TESTS=no
-  fast_finish: true
-  include:
-    - go: 1.11.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.12.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.13.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.14.x
-      env: RUN_LONG_TESTS=yes
-    - go: 1.15.x
-      env:
-        - RUN_LONG_TESTS=yes
-        - DEPLOY_BINARIES=yes
-        - APTLY_USER=aptly
-        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
-    - go: master
-      env: RUN_LONG_TESTS=no
-
-before_install:
-  - virtualenv system/env
-  - . system/env/bin/activate
-  - pip install six packaging appdirs
-  - pip install -U pip setuptools
-  - pip install -r system/requirements.txt
-  - make version
-
-install:
-  - make prepare
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-notifications:
-  webhooks:
-    urls:
-      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false
-
-before_deploy:
-  - make release
-
-deploy:
-  - provider: releases
-    api_key:
-      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
-    file_glob: true
-    file: build/*
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: s3
-    access_key_id:
-      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
-    secret_access_key:
-      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
-    bucket: aptly-nightly
-    skip_cleanup: true
-    acl: public-read
-    local_dir: build
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh nightly
-    skip_cleanup: true
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh release
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -49,3 +49,22 @@ List of contributors, in chronological order:
 * Samuel Mutel (https://github.com/smutel)
 * Russell Greene (https://github.com/russelltg)
 * Wade Simmons (https://github.com/wadey)
+* Steven Stone (https://github.com/smstone)
+* Josh Bayfield (https://github.com/jbayfield)
+* Boxjan (https://github.com/boxjan)
+* Mauro Regli (https://github.com/reglim)
+* Alexander Zubarev (https://github.com/strike)
+* Nicolas Dostert (https://github.com/acdn-ndostert)
+* Ryan Gonzalez (https://github.com/refi64)
+* Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramon N.Rodriguez (https://github.com/runitonmetal)
+* Golf Hu (https://github.com/hudeng-go)
+* Cookie Fei (https://github.com/wuhuang26)
+* Andrey Loukhnov (https://github.com/aol-nnov)
+* Christoph Fiehe (https://github.com/cfiehe)
+* Blake Kostner (https://github.com/btkostner)
+* Leigh London (https://github.com/leighlondon)
+* Gordian Schoenherr (https://github.com/schoenherrg)

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 :+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
 
-The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
 These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
 ## What should I know before I get started?
@@ -11,7 +11,7 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
 
 This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
 By participating, you are expected to uphold this code.
-Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
 
 ### List of Repositories
 
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your First Code Contribution
+### Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,63 +68,117 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
 Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
 if that is possible.
 
+### Get the Source
+
+To clone the git repo, run the following commands:
+```
+git clone git@github.com:aptly-dev/aptly.git
+cd aptly
+```
+
 ## Development Setup
 
-This section describes local setup to start contributing to aptly source.
+Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
 
-### Go & Python
+### Docker Development Setup
 
-You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+This section describes the docker setup to start contributing to aptly.
 
-If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
-initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+#### Dependencies
 
-Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
-to prepend it or to skip this test if you're security conscious.
+Install the following on your development machine:
+- docker
+- make
+- git
 
-### Forking and Cloning
+##### Docker installation on macOS
+1. Install [Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install/) (or via [Homebrew](https://brew.sh/))
+2. Allow directory sharing
+   - Open Docker Desktop
+   - Go to `Settings → Resources → File Sharing → Virtual File Shares`
+   - Add the aptly git repository path to the shared list (eg. /home/Users/john/aptly)
 
-As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
+#### Create docker container
 
-    git clone git@github.com:aptly-dev/aptly.git
-    cd aptly
+To build the development docker image, run:
+```
+make docker-image
+```
 
-For main repo under your GitHub user and add it as another Git remote:
+#### Build aptly
 
-    git remote add <user> git@github.com:<user>/aptly.git
+To build the aptly in the development docker container, run:
+```
+make docker-build
+```
 
-That way you can continue to build project as is (you don't need to adjust import paths), but you would need
-to specify your remote name when pushing branches:
+#### Running aptly commands
 
-    git push <user> <your-branch>
+To run aptly commands in the development docker container, run:
+```
+make docker-shell
+```
 
-### Dependencies
+Example:
+```
+$ make docker-shell
+aptly@b43e8473ef81:/work/src$ aptly version
+aptly version: 1.5.0+189+g0fc90dff
+```
 
-You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+#### Running unit tests
 
-    make prepare dev system/env
+In order to run aptly unit tests, enter the following:
+```
+make docker-unit-tests
+```
 
-This is usually one-time action.
+#### Running system tests
 
-Aptly is using Go modules to manage dependencies, download modules using:
+In order to run aptly system tests, enter the following:
+```
+make docker-system-tests
+```
 
-    make modules
+#### Running golangci-lint
 
-### Building
+In order to run aptly unit tests, run:
+```
+make docker-lint
+```
 
-If you want to build aptly binary from your current source tree, run:
+#### More info
+
+Run `make help` for more information.
+
+
+### Local Development Setup
+
+This section describes local setup to start contributing to aptly.
+
+#### Dependencies
+
+Building aptly requires go version 1.22.
+
+On Debian bookworm with backports enabled, go can be installed with:
+
+    apt install -t bookworm-backports golang-go
+
+#### Building
+
+To build aptly, run:
+
+    make build
+
+Run aptly:
+
+    build/aptly
+
+To install aptly into `$GOPATH/bin`, run:
 
     make install
 
-This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
-
-    aptly
-
-Or, if it's not on your path:
-
-    ~/go/bin/aptly
-
-### Unit-tests
+#### Unit-tests
 
 aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
 feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -133,7 +187,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
 
     make test
 
-### Functional Tests
+#### Functional Tests
 
 Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
 runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -180,26 +234,6 @@ There are some packages available under `system/files/` directory which are used
 this default location. You can run aptly under different user or by using non-default config location with non-default
 aptly root directory.
 
-### Style Checks
-
-Style checks could be run with:
-
-    make check
-
-aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
-for the linter could be found in [.golangci.yml](.golangci.yml) file.
-
-Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
-
-### Vendored Code
-
-aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
-repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
-packages over packages in `$GOPATH`.
-
-If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
-Usually all you need is `dep ensure` or `dep ensure -update`.
-
 ### man Page
 
 aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
@@ -217,25 +251,3 @@ Bash and Zsh completion for aptly reside in the same repo under in [completion.d
 When new option or command is introduced, bash completion should be updated to reflect that change.
 
 When aptly package is being built, it automatically pulls bash completion and man page into the package.
-
-## Design
-
-This section requires future work.
-
-*TBD*
-
-### Database
-
-### Package Pool
-
-### Package
-
-### PackageList, PackageRefList
-
-### LocalRepo, RemoteRepo, Snapshot
-
-### PublishedRepository
-
-### Context
-
-### Collections, CollectionFactory

Makefile

@@ -1,81 +1,219 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
-ifdef TRAVIS_TAG
-	TAG=$(TRAVIS_TAG)
-else
-	TAG="$(shell git describe --tags --always)"
-endif
-VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files gpg http query swift s3 utils
+GOPATH=$(shell go env GOPATH)
+VERSION=$(shell make -s version)
 PYTHON?=python3
-TESTS?=
 BINPATH?=$(GOPATH)/bin
-RUN_LONG_TESTS?=yes
+GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-all: modules test bench check system-test
+# Uncomment to update system test gold files
+# CAPTURE := "--capture"
 
-prepare:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-modules:
-	go mod download
+prepare:  ## Install go module dependencies
+	# Prepare go modules
 	go mod verify
 	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-dev:
-	go get -u github.com/laher/goxc
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
+	@reltype=ci ; \
+	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
+	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
+		gittag=`git describe --tags --exact-match 2>/dev/null` ;\
+		if echo "$$gittag" | grep -q '^v[0-9]'; then \
+			reltype=release ; \
+		fi ; \
+	fi ; \
+	echo $$reltype
 
-check: system/env
-ifeq ($(RUN_LONG_TESTS), yes)
-	golangci-lint run
-	system/env/bin/flake8
-endif
+version:  ## Print aptly version
+	@ci="" ; \
+	if [ "`make -s releasetype`" = "ci" ]; then \
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
+	else \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
+	fi
+
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
+	# Generate swagger.conf
+	cp docs/swagger.conf.tpl docs/swagger.conf
+	echo "// @version $(VERSION)" >> docs/swagger.conf
+
+azurite-start:
+	azurite -l /tmp/aptly-azurite & \
+	echo $$! > ~/.azurite.pid
+
+azurite-stop:
+	@kill `cat ~/.azurite.pid`
+
+swagger: swagger-install
+	# Generate swagger docs
+	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+
+etcd-install:
+	# Install etcd
+	test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint: prepare
+	# Install golangci-lint
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
 
 install:
-	go install -v -ldflags "-X main.Version=$(VERSION)"
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	# go generate
+	@go generate
+	# go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system/env: system/requirements.txt
-ifeq ($(RUN_LONG_TESTS), yes)
-	rm -rf system/env
-	$(PYTHON) -m venv system/env
-	system/env/bin/pip install -r system/requirements.txt
-endif
+test: prepare swagger etcd-install  ## Run unit tests
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
+	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
+	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/aptly-etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
 
-system-test: install system/env
-ifeq ($(RUN_LONG_TESTS), yes)
+system-test: prepare swagger etcd-install  ## Run system tests
+	# build coverage binary
+	go test -v -coverpkg="./..." -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
-endif
-
-test:
-	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
+	# Run system tests
+	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
 bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
 	go test -v ./deb -run=nothing -bench=. -benchmem
 
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
+
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
+	fi
+	# clean
+	rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
+	# Run dpkg-buildpackage
+	@buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
+	echo "$$cmd" ; \
+	$$cmd
+	lintian ../*_$(DEBARCH).changes || true
+	# cleanup
+	@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-test:  ## Run unit tests in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		test \
+		azurite-stop
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		system-test TEST=$(TEST) \
+		azurite-stop
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
+
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc: dev
-	rm -rf root/
-	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
-	cp man/aptly.1 root/usr/share/man/man1
-	cp completion.d/aptly root/etc/bash_completion.d/
-	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
-	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
-
-release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
-release: goxc
-	rm -rf build/
-	mkdir -p build/
-	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
-
-man:
+man:  ## Create man pages
 	make -C man
 
-version:
-	@echo $(VERSION)
+clean:  ## remove local build and module cache
+	# Clean all generated and build files
+	test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
+	rm -rf .go/
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: man modules version release goxc
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -1,7 +1,3 @@
-=====
-aptly
-=====
-
 .. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
     :target: https://github.com/aptly-dev/aptly/actions
 
@@ -9,20 +5,23 @@ aptly
     :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly
 
+aptly
+=====
+
 Aptly is a swiss army knife for Debian repository management.
 
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
-mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
+open `issues <https://github.com/aptly-dev/aptly/issues>`_ or `discussions <https://github.com/aptly-dev/aptly/discussions>`_.
 
-Aptly features: ("+" means planned features)
+Aptly features:
 
 * make mirrors of remote Debian/Ubuntu repositories, limiting by components/architectures
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
@@ -32,53 +31,70 @@ Aptly features: ("+" means planned features)
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
 * REST API for remote access
-* mirror repositories "as-is" (without resigning with user's key) (+)
-* support for yum repositories (+)
 
-Current limitations:
+Any contributions are welcome! Please see `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
-* translations are not supported yet
+Installation
+=============
 
-Download
---------
+Aptly can be installed on several operating systems.
 
-To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
+Debian / Ubuntu
+----------------
 
-    deb http://repo.aptly.info/ squeeze main
+Aptly is provided in the following debian packages:
 
-And import key that is used to sign the release::
+* **aptly**: Includes the main Aptly binary, man pages, and shell completions
+* **aptly-api**: A systemd service for the REST API, using the global /etc/aptly.conf
+* **aptly-dbg**: Debug symbols for troubleshooting
 
-    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
+The packages can be installed on official `Debian <https://packages.debian.org/search?keywords=aptly>`_ and `Ubuntu <https://packages.ubuntu.com/search?keywords=aptly>`_ distributions.
 
-After that you can install aptly as any other software package::
+Upstream Debian Packages
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    $ apt-get update
-    $ apt-get install aptly
+If a newer version (not available in Debian/Ubuntu) of aptly is required, upstream debian packages (built from git tags) can be installed as follows:
 
-Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
-Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+Install the following APT key (as root)::
 
-If you would like to use nightly builds (unstable), please use following repository::
+    wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
 
-    deb http://repo.aptly.info/ nightly main
+Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
 
-Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
 
-If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
-    git clone https://github.com/aptly-dev/aptly
-    cd aptly
-    make modules install
+Install aptly packages::
 
-Binary would be installed to ``$GOPATH/bin/aptly``.
+    apt-get update
+    apt-get install aptly
+    apt-get install aptly-api  # REST API systemd service
 
-Contributing
-------------
+CI Builds
+~~~~~~~~~~
 
-Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+For testing new features or bugfixes, recent builds are available as CI builds (built from master, may be unstable!) and can be installed as follows:
+
+Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
+
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
+
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
+
+Note: same gpg key is used as for the Upstream Debian Packages.
+
+Other Operating Systems
+------------------------
+
+Binary executables (depends almost only on libc) are available on `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_ for:
+
+- macOS / darwin (amd64, arm64)
+- FreeBSD (amd64, arm64, 386, arm)
+- Generic Linux (amd64, arm64, 386, arm)
 
 Integrations
-------------
+=============
 
 Vagrant:
 
@@ -115,3 +131,7 @@ GUI for aptly API:
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar
+
+Molior:
+
+-   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth

Releasing.md

@@ -0,0 +1,16 @@
+# Creating a Release
+
+- create branch release/1.x.y
+- update debian/changelog
+- create PR, merge when approved
+- on updated master, create release:
+```
+version=$(dpkg-parsechangelog -S Version)
+echo Releasing prod version $version
+git tag -a v$version -m 'aptly: release $version'
+git push origin v$version master
+```
+- run swagger locally (`make docker-serve`)
+- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
+- releae www.aptly.info
+- create release announcement on https://github.com/aptly-dev/aptly/discussions

api/api.go

@@ -3,17 +3,18 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
 // Lock order acquisition (canonical):
@@ -22,11 +23,67 @@ import (
 //  3. SnapshotCollection
 //  4. PublishedRepoCollection
 
-// GET /api/version
+type aptlyVersion struct {
+	// Aptly Version
+	Version string `json:"Version"`
+}
+
+// @Summary Aptly Version
+// @Description **Get aptly version**
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/version
+// @Description {"Version":"0.9~dev"}
+// @Description ```
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyVersion
+// @Router /api/version [get]
 func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+type aptlyStatus struct {
+	// Aptly Status
+	Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"`
+}
+
+// @Summary Get Ready State
+// @Description **Get aptly ready state**
+// @Description
+// @Description Return aptly ready state:
+// @Description - `Aptly is ready` (HTTP 200)
+// @Description - `Aptly is unavailable` (HTTP 503)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus "Aptly is ready"
+// @Failure 503 {object} aptlyStatus "Aptly is unavailable"
+// @Router /api/ready [get]
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+	}
+}
+
+// @Summary Get Health State
+// @Description **Get aptly health state**
+// @Description
+// @Description Return aptly health state:
+// @Description - `Aptly is healthy` (HTTP 200)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus
+// @Router /api/healthy [get]
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
+}
+
 type dbRequestKind int
 
 const (
@@ -137,20 +194,29 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 	// Run this task in background if configured globally or per-request
 	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
 	if background {
-		log.Println("Executing task asynchronously")
+		log.Debug().Msg("Executing task asynchronously")
 		task, conflictErr := runTaskInBackground(name, resources, proc)
 		if conflictErr != nil {
-			c.AbortWithError(409, conflictErr)
+			AbortWithJSONError(c, 409, conflictErr)
 			return
 		}
 		c.JSON(202, task)
 	} else {
-		log.Println("Executing task synchronously")
-		out := context.Progress()
-		detail := task.Detail{}
-		retValue, err := proc(out, &detail)
+		log.Debug().Msg("Executing task synchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
+		context.TaskList().DeleteTaskByID(task.ID)
 		if err != nil {
-			c.AbortWithError(retValue.Code, err)
+			AbortWithJSONError(c, retValue.Code, err)
 			return
 		}
 		if retValue != nil {
@@ -168,7 +234,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -176,7 +242,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -193,21 +259,57 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			Source:            nil,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter(deb.FilterOptions{
+					Queries: []deb.PackageQuery{versionQ},
+				})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
 		list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
@@ -219,3 +321,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 		c.JSON(200, list.Strings())
 	}
 }
+
+func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+	return c.AbortWithError(code, err)
+}

api/api_test.go

@@ -1,16 +1,20 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
-	ctx "github.com/aptly-dev/aptly/context"
-	"github.com/gin-gonic/gin"
+	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
 	"github.com/smira/flag"
 
 	. "gopkg.in/check.v1"
@@ -30,12 +34,13 @@ type ApiSuite struct {
 var _ = Suite(&ApiSuite{})
 
 func createTestConfig() *os.File {
-	file, err := ioutil.TempFile("", "aptly")
+	file, err := os.CreateTemp("", "aptly")
 	if err != nil {
 		return nil
 	}
 	jsonString, err := json.Marshal(gin.H{
-		"architectures": []string{},
+		"architectures":         []string{},
+		"enableMetricsEndpoint": true,
 	})
 	if err != nil {
 		return nil
@@ -44,9 +49,12 @@ func createTestConfig() *os.File {
 	return file
 }
 
-func (s *ApiSuite) SetUpSuite(c *C) {
+func (s *ApiSuite) setupContext() error {
+	aptly.Version = "testVersion"
 	file := createTestConfig()
-	c.Assert(file, NotNil)
+	if nil == file {
+		return fmt.Errorf("unable to create the test configuration file")
+	}
 	s.configFile = file
 
 	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
@@ -57,10 +65,19 @@ func (s *ApiSuite) SetUpSuite(c *C) {
 	s.flags = flags
 
 	context, err := ctx.NewContext(s.flags)
-	c.Assert(err, IsNil)
+	if nil != err {
+		return err
+	}
 
 	s.context = context
 	s.router = Router(context)
+
+	return nil
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	err := s.setupContext()
+	c.Assert(err, IsNil)
 }
 
 func (s *ApiSuite) TearDownSuite(c *C) {
@@ -85,11 +102,52 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
 	return w, nil
 }
 
+func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
+}
+
 func (s *ApiSuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, ".*Version.*")
+	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
+}
+
+func (s *ApiSuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *ApiSuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *ApiSuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
+}
+
+func (s *ApiSuite) TestRepoCreate(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name": "dummy",
+	})
+	c.Assert(err, IsNil)
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
 }
 
 func (s *ApiSuite) TestTruthy(c *C) {

api/db.go

@@ -11,9 +11,17 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// POST /api/db/cleanup
+// @Summary DB Cleanup
+// @Description **Cleanup Aptly DB**
+// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore.
+// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos.
+// @Tags Database
+// @Produce json
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/db/cleanup [post]
 func apiDbCleanup(c *gin.Context) {
-
 	resources := []string{string(task.AllResourcesKey)}
 	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 		var err error

api/error.go

@@ -0,0 +1,5 @@
+package api
+
+type Error struct {
+	Error string `json:"error"`
+}

api/files.go

@@ -6,8 +6,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
 func verifyPath(path string) bool {
@@ -24,27 +27,37 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
 	return true
 }
 
-// GET /files
+// @Summary List Directories
+// @Description **Get list of upload directories**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description  $ curl http://localhost:8080/api/files
+// @Description  ["aptly-0.9"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Success 200 {array} string "List of files"
+// @Router /api/files [get]
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -53,30 +66,50 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, list)
 }
 
-// POST /files/:dir/
+// @Summary Upload Files
+// @Description **Upload files to a directory**
+// @Description
+// @Description - one or more files can be uploaded
+// @Description - existing uploaded are overwritten
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Accept multipart/form-data
+// @Param dir path string true "Directory to upload files to. Created if does not exist"
+// @Param files formData file true "Files to upload"
+// @Produce json
+// @Success 200 {array} string "list of uploaded files"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [post]
 func apiFilesUpload(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -86,7 +119,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +127,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -109,20 +142,35 @@ func apiFilesUpload(c *gin.Context) {
 		}
 	}
 
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
-
 }
 
-// GET /files/:dir
+// @Summary List Files
+// @Description **Show uploaded files in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to list"
+// @Success 200 {array} string "Files found in directory"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [get]
 func apiFilesListFiles(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
 	list := []string{}
-	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	listLock := &sync.Mutex{}
+	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -131,6 +179,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil
@@ -138,9 +188,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 		} else {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 		}
 		return
 	}
@@ -148,36 +198,66 @@ func apiFilesListFiles(c *gin.Context) {
 	c.JSON(200, list)
 }
 
-// DELETE /files/:dir
+// @Summary Delete Directory
+// @Description **Delete upload directory and uploaded files within**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [delete]
 func apiFilesDeleteDir(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, gin.H{})
 }
 
-// DELETE /files/:dir/:name
+// @Summary Delete File
+// @Description **Delete a uploaded file in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to delete from"
+// @Param name path string true "File to delete"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir}/{name} [delete]
 func apiFilesDeleteFile(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	if !verifyPath(c.Params.ByName("name")) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+	dir := utils.SanitizePath(c.Params.ByName("dir"))
+	name := utils.SanitizePath(c.Params.ByName("name"))
+	if !verifyPath(name) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 	}

api/gpg.go

@@ -2,31 +2,49 @@ package api
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
 
-// POST /api/gpg
-func apiGPGAddKey(c *gin.Context) {
-	var b struct {
-		Keyserver   string
-		GpgKeyID    string
-		GpgKeyArmor string
-		Keyring     string
-	}
+type gpgAddKeyParams struct {
+	// Keyserver, when downloading GpgKeyIDs
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// GpgKeyIDs to download from Keyserver, comma separated list
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
+	// Armored gpg public ket, instead of downloading from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+	// Keyring for adding the keys (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+}
 
+// @Summary Add GPG Keys
+// @Description **Adds GPG keys to aptly keyring**
+// @Description
+// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/gpg [post]
+func apiGPGAddKey(c *gin.Context) {
+	b := gpgAddKeyParams{}
 	if c.Bind(&b) != nil {
 		return
 	}
+	b.Keyserver = utils.SanitizePath(b.Keyserver)
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
+	// b.Keyring can be an absolute path
 
 	var err error
-	args := []string{"--no-default-keyring"}
+	args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
 	keyring := "trustedkeys.gpg"
 	if len(b.Keyring) > 0 {
 		keyring = b.Keyring
@@ -37,9 +55,9 @@ func apiGPGAddKey(c *gin.Context) {
 	}
 	if len(b.GpgKeyArmor) > 0 {
 		var tempdir string
-		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 		defer os.RemoveAll(tempdir)
@@ -47,11 +65,11 @@ func apiGPGAddKey(c *gin.Context) {
 		keypath := filepath.Join(tempdir, "key")
 		keyfile, e := os.Create(keypath)
 		if e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 			return
 		}
 		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 		}
 		args = append(args, "--import", keypath)
 
@@ -62,10 +80,10 @@ func apiGPGAddKey(c *gin.Context) {
 		args = append(args, keys...)
 	}
 
-	finder := pgp.GPG1Finder()
+	finder := pgp.GPGDefaultFinder()
 	gpg, _, err := finder.FindGPG()
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -74,11 +92,11 @@ func apiGPGAddKey(c *gin.Context) {
 	// there is no error handling for such as gpg will do this for us
 	cmd := exec.Command(gpg, args...)
 	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
-	cmd.Stdout = os.Stdout
-	if err = cmd.Run(); err != nil {
-		c.AbortWithError(400, err)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		c.JSON(400, string(out))
 		return
 	}
 
-	c.JSON(200, gin.H{})
+	c.JSON(200, string(out))
 }

api/graph.go

@@ -12,7 +12,27 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+// @Summary Graph Output
+// @Description **Generate dependency graph**
+// @Description
+// @Description Command graph generates graph of dependencies:
+// @Description
+// @Description * between snapshots and mirrors (what mirror was used to create each snapshot)
+// @Description * between snapshots and local repos (what local repo was used to create snapshot)
+// @Description * between snapshots (pulling, merging, etc.)
+// @Description * between snapshots, local repos and published repositories (how snapshots were published).
+// @Description
+// @Description Graph is rendered to PNG file using graphviz package.
+// @Description
+// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical`
+// @Tags Status
+// @Produce image/png, image/svg+xml
+// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg."
+// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout."
+// @Success 200 {object} []byte "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/graph.{ext} [get]
 func apiGraph(c *gin.Context) {
 	var (
 		err    error
@@ -43,25 +63,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/metrics.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+	apiFilesUploadedCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_files_uploaded_total",
+			Help: "Total number of uploaded files labeled by upload directory.",
+		},
+		[]string{"directory"},
+	)
+	apiReposPackageCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_repos_package_count",
+			Help: "Current number of published packages labeled by source, distribution and component.",
+		},
+		[]string{"source", "distribution", "component"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
+
+func countPackagesByRepos() {
+	err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if err != nil {
+			msg := fmt.Sprintf(
+				"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
+				err, repo.StoragePrefix(), repo.Distribution, repo.Components())
+			log.Warn().Msg(msg)
+			return err
+		}
+
+		components := repo.Components()
+		for _, c := range components {
+			count := float64(len(repo.RefList(c).Refs))
+			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
+		log.Warn().Msg(msg)
+	}
+}

api/middleware.go

@@ -9,59 +9,36 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	apiRequestsInFlightGauge = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "aptly_api_http_requests_in_flight",
-			Help: "Number of concurrent HTTP api requests currently handled.",
-		},
-		[]string{"method", "path"},
-	)
-	apiRequestsTotalCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "aptly_api_http_requests_total",
-			Help: "Total number of api requests.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_size_bytes",
-			Help: "Api HTTP request size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiResponseSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_response_size_bytes",
-			Help: "Api HTTP response size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestsDurationSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_duration_seconds",
-			Help: "Duration of api requests in seconds.",
-		},
-		[]string{"code", "method", "path"},
-	)
+	"github.com/rs/zerolog/log"
 )
 
 // Only use base path as label value (e.g.: /api/repos) because of time series cardinality
 // See https://prometheus.io/docs/practices/naming/#labels
 func getBasePath(c *gin.Context) string {
-	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+	segment0, err := getURLSegment(c.Request.URL.Path, 0)
+	if err != nil {
+		return "/"
+	}
+	segment1, err := getURLSegment(c.Request.URL.Path, 1)
+	if err != nil {
+		return *segment0
+	}
+
+	return *segment0 + *segment1
 }
 
-func getURLSegment(url string, idx int) string {
-	var urlSegments = strings.Split(url, "/")
-
+func getURLSegment(url string, idx int) (*string, error) {
+	urlSegments := strings.Split(url, "/")
 	// Remove segment at index 0 because it's an empty string
-	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
-	return fmt.Sprintf("/%s", segmentAtIndex)
+	urlSegments = urlSegments[1:cap(urlSegments)]
+
+	if len(urlSegments) <= idx {
+		return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
+	}
+
+	segmentAtIndex := urlSegments[idx]
+	s := fmt.Sprintf("/%s", segmentAtIndex)
+	return &s, nil
 }
 
 func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
@@ -101,3 +78,39 @@ func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Co
 		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
 	}
 }
+
+// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
+// logs that include error messages if there are any.
+func JSONLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
+
+		// Process request
+		c.Next()
+
+		ts := time.Now()
+		if raw != "" {
+			path = path + "?" + raw
+		}
+
+		errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
+		l := log.With().Str("remote", c.ClientIP()).Logger().
+			With().Str("method", c.Request.Method).Logger().
+			With().Str("path", path).Logger().
+			With().Str("protocol", c.Request.Proto).Logger().
+			With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
+			With().Str("latency", ts.Sub(start).String()).Logger().
+			With().Str("agent", c.Request.UserAgent()).Logger()
+
+		if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
+			l.Warn().Msg(errorMessage)
+		} else if c.Writer.Status() >= 500 {
+			l.Error().Msg(errorMessage)
+		} else {
+			l.Info().Msg(errorMessage)
+		}
+	}
+}

api/middleware_test.go

@@ -0,0 +1,255 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MiddlewareSuite struct {
+	router    http.Handler
+	context   *gin.Context
+	logReader *os.File
+	logWriter *os.File
+}
+
+var _ = Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) SetUpTest(c *C) {
+	r, w, err := os.Pipe()
+	c.Assert(err, IsNil)
+
+	utils.SetupJSONLogger("debug", w)
+	mw := JSONLogger()
+
+	router := gin.New()
+	router.UseRawPath = true
+	router.Use(mw)
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	root := router.Group("/api")
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	root.GET("/ready", apiReady(isReady))
+	root.GET("/healthy", apiHealthy)
+
+	s.router = router
+	s.logReader = r
+	s.logWriter = w
+}
+
+func (s *MiddlewareSuite) TearDownTest(c *C) {
+	s.router = nil
+	s.context = nil
+	s.logReader = nil
+	s.logWriter = nil
+}
+
+func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
+	recorder := httptest.NewRecorder()
+	s.context, _ = gin.CreateTestContext(recorder)
+	req, _ := http.NewRequestWithContext(s.context, method, url, body)
+	s.context.Request = req
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "warn")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["method"]; ok {
+		c.Check(val, Equals, "GET")
+	} else {
+		c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["path"]; ok {
+		c.Check(val, Equals, "/")
+	} else {
+		c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["protocol"]; ok {
+		c.Check(val, Equals, "HTTP/1.1")
+	} else {
+		c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["code"]; ok {
+		c.Check(val, Equals, "404")
+	} else {
+		c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["remote"]; !ok {
+		c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["latency"]; !ok {
+		c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["agent"]; !ok {
+		c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["time"]; !ok {
+		c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "error")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	fmt.Println(capturedOutput)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestGetBasePath(c *C) {
+	s.HTTPRequest(http.MethodGet, "", nil)
+	path := getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/api", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api")
+
+	s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api/repos")
+}
+
+func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
+	url := "/"
+	segment, err := getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api/repos/testRepo"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	segment, err = getURLSegment(url, 1)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/repos")
+}

api/mirror.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
+	"os"
 	"sort"
 	"strings"
 	"sync"
@@ -14,19 +14,16 @@ import (
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
-func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
-	if ignoreSignatures {
-		return nil, nil
-	}
-
+func getVerifier(keyRings []string) (pgp.Verifier, error) {
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(false)
 	if err != nil {
 		return nil, err
 	}
@@ -34,7 +31,13 @@ func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error)
 	return verifier, nil
 }
 
-// GET /api/mirrors
+// @Summary List Mirrors
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {array} deb.RemoteRepo
+// @Router /api/mirrors [get]
 func apiMirrorsList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -48,24 +51,49 @@ func apiMirrorsList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// POST /api/mirrors
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
+// @Summary Create Mirror
+// @Description **Create a mirror of a remote repository**
+// @Tags Mirrors
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/mirrors [post]
 func apiMirrorsCreate(c *gin.Context) {
 	var err error
-	var b struct {
-		Name               string `binding:"required"`
-		ArchiveURL         string `binding:"required"`
-		Distribution       string
-		Filter             string
-		Components         []string
-		Architectures      []string
-		Keyrings           []string
-		DownloadSources    bool
-		DownloadUdebs      bool
-		DownloadInstaller  bool
-		FilterWithDeps     bool
-		SkipComponentCheck bool
-		IgnoreSignatures   bool
-	}
+	var b mirrorCreateParams
 
 	b.DownloadSources = context.Config().DownloadSourcePackages
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -81,7 +109,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
 		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 	}
@@ -89,7 +117,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	if b.Filter != "" {
 		_, err = query.Parse(b.Filter)
 		if err != nil {
-			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 			return
 		}
 	}
@@ -98,39 +126,51 @@ func apiMirrorsCreate(c *gin.Context) {
 		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
 
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 		return
 	}
 
 	repo.Filter = b.Filter
 	repo.FilterWithDeps = b.FilterWithDeps
 	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.SkipArchitectureCheck = b.SkipArchitectureCheck
 	repo.DownloadSources = b.DownloadSources
 	repo.DownloadUdebs = b.DownloadUdebs
 
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
 	downloader := context.NewDownloader(nil)
-	err = repo.Fetch(downloader, verifier)
+	err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
 		return
 	}
 
 	err = collection.Add(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
 		return
 	}
 
 	c.JSON(201, repo)
 }
 
-// DELETE /api/mirrors/:name
+// @Summary Delete Mirror
+// @Description **Delete a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param force query int true "force: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
+// @Failure 500 {object} Error "Unable to delete"
+// @Router /api/mirrors/{name} [delete]
 func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
@@ -141,13 +181,13 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
@@ -157,7 +197,7 @@ func apiMirrorsDrop(c *gin.Context) {
 			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
@@ -169,7 +209,15 @@ func apiMirrorsDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/mirrors/:name
+// @Summary Get Mirror Info
+// @Description **Get mirror information by name**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [get]
 func apiMirrorsShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -177,19 +225,30 @@ func apiMirrorsShow(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
 
 	c.JSON(200, repo)
 }
 
-// GET /api/mirrors/:name/packages
+// @Summary List Mirror Packages
+// @Description **Get a list of packages from a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Produce json
+// @Success 200 {array} deb.Package "List of Packages"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name}/packages [get]
 func apiMirrorsPackages(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -197,17 +256,17 @@ func apiMirrorsPackages(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
 
 	if repo.LastDownloadDate.IsZero() {
-		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
 		return
 	}
 
@@ -216,7 +275,7 @@ func apiMirrorsPackages(c *gin.Context) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -224,7 +283,7 @@ func apiMirrorsPackages(c *gin.Context) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -241,17 +300,21 @@ func apiMirrorsPackages(c *gin.Context) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 		}
 	}
 
@@ -267,36 +330,66 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 }
 
-// PUT /api/mirrors/:name
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Url of the archive to mirror
+	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
+	// Package query that is applied to mirror packages
+	Filter string `               json:"Filter"                 example:"xserver-xorg"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `      json:"Architectures"          example:"amd64"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `         json:"Components"             example:"main"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `         json:"FilterWithDeps"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `        json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `          json:"DownloadUdebs"`
+	// Set "true" to skip checking if the given components are in the Release file
+	SkipComponentCheck bool `     json:"SkipComponentCheck"`
+	// Set "true" to skip checking if the given architectures are in the Release file
+	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+}
+
+// @Summary Update Mirror
+// @Description **Update Mirror and download packages**
+// @Tags Mirrors
+// @Param name path string true "mirror name to update"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [put]
 func apiMirrorsUpdate(c *gin.Context) {
 	var (
 		err    error
 		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
 	)
 
-	var b struct {
-		Name                 string
-		ArchiveURL           string
-		Filter               string
-		Architectures        []string
-		Components           []string
-		Keyrings             []string
-		FilterWithDeps       bool
-		DownloadSources      bool
-		DownloadUdebs        bool
-		SkipComponentCheck   bool
-		IgnoreChecksums      bool
-		IgnoreSignatures     bool
-		ForceUpdate          bool
-		SkipExistingPackages bool
-	}
-
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
 	remote, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -304,12 +397,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	b.DownloadUdebs = remote.DownloadUdebs
 	b.DownloadSources = remote.DownloadSources
 	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
 	b.FilterWithDeps = remote.FilterWithDeps
 	b.Filter = remote.Filter
 	b.Architectures = remote.Architectures
 	b.Components = remote.Components
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
-	log.Printf("%s: Starting mirror update\n", b.Name)
+	log.Info().Msgf("%s: Starting mirror update", b.Name)
 
 	if c.Bind(&b) != nil {
 		return
@@ -318,14 +413,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
-			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
 			return
 		}
 	}
 
 	if b.DownloadUdebs != remote.DownloadUdebs {
 		if remote.IsFlat() && b.DownloadUdebs {
-			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
 			return
 		}
 	}
@@ -338,14 +433,15 @@ func apiMirrorsUpdate(c *gin.Context) {
 	remote.DownloadUdebs = b.DownloadUdebs
 	remote.DownloadSources = b.DownloadSources
 	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
 	remote.FilterWithDeps = b.FilterWithDeps
 	remote.Filter = b.Filter
 	remote.Architectures = b.Architectures
 	remote.Components = b.Components
 
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
@@ -353,7 +449,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier)
+		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -365,7 +461,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -458,7 +554,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}()
 
-		log.Printf("%s: Spawning background processes...\n", b.Name)
+		log.Info().Msgf("%s: Spawning background processes...", b.Name)
 		var wg sync.WaitGroup
 		for i := 0; i < context.Config().DownloadConcurrency; i++ {
 			wg.Add(1)
@@ -476,7 +572,16 @@ func apiMirrorsUpdate(c *gin.Context) {
 						var e error
 
 						// provision download location
-						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+							task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+						} else {
+							var file *os.File
+							file, e = os.CreateTemp("", task.File.Filename)
+							if e == nil {
+								task.TempDownPath = file.Name()
+								file.Close()
+							}
+						}
 						if e != nil {
 							pushError(e)
 							continue
@@ -494,6 +599,20 @@ func apiMirrorsUpdate(c *gin.Context) {
 							continue
 						}
 
+						// and import it back to the pool
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						if err != nil {
+							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+							pushError(err)
+							continue
+						}
+
+						// update "attached" files if any
+						for _, additionalAtask := range task.Additional {
+							additionalAtask.File.PoolPath = task.File.PoolPath
+							additionalAtask.File.Checksums = task.File.Checksums
+						}
+
 						task.Done = true
 						taskFinished <- task
 					case <-context.Done():
@@ -505,32 +624,22 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		// Wait for all download goroutines to finish
-		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
 		wg.Wait()
-		log.Printf("%s: Background processes finished\n", b.Name)
+		log.Info().Msgf("%s: Background processes finished", b.Name)
 		close(taskFinished)
 
-		for idx := range queue {
+		defer func() {
+			for _, task := range queue {
+				if task.TempDownPath == "" {
+					continue
+				}
 
-			atask := &queue[idx]
-
-			if !atask.Done {
-				// download not finished yet
-				continue
+				if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+					fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+				}
 			}
-
-			// and import it back to the pool
-			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
-			}
-
-			// update "attached" files if any
-			for _, additionalAtask := range atask.Additional {
-				additionalAtask.File.PoolPath = atask.File.PoolPath
-				additionalAtask.File.Checksums = atask.File.Checksums
-			}
-		}
+		}()
 
 		select {
 		case <-context.Done():
@@ -539,18 +648,18 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		if len(errors) > 0 {
-			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 		}
 
-		log.Printf("%s: Finalizing download\n", b.Name)
+		log.Info().Msgf("%s: Finalizing download...", b.Name)
 		remote.FinalizeDownload(collectionFactory, out)
 		err = collectionFactory.RemoteRepoCollection().Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		log.Info().Msgf("%s: Mirror updated successfully", b.Name)
 		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
 	})
 }

api/packages.go

@@ -1,17 +1,41 @@
 package api
 
 import (
+	_ "github.com/aptly-dev/aptly/deb" // for swagger
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/packages/:key
+// @Summary Get Package Info
+// @Description **Show information about package by package key**
+// @Description Package keys could be obtained from various GET .../packages APIs.
+// @Tags Packages
+// @Produce json
+// @Param key path string true "package key (unique package identifier)"
+// @Success 200 {object} deb.Package "OK"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/packages/{key} [get]
 func apiPackagesShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, p)
 }
+
+// @Summary List Packages
+// @Description **Get list of packages**
+// @Tags Packages
+// @Consume  json
+// @Produce  json
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Success 200 {array} string "List of packages"
+// @Router /api/packages [get]
+func apiPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PackageCollection()
+	showPackages(c, collection.AllPackageRefs(), collectionFactory)
+}

api/packages_test.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type PackagesSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&PackagesSuite{})
+
+func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}

api/repos.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -17,7 +18,59 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/repos
+// @Summary Serve HTML Listing
+// @Description If ServeInAPIMode is enabled in aptly config,
+// @Description this endpoint is enabled which returns an HTML listing of each repo that can be browsed
+// @Tags Repos
+// @Produce html
+// @Success 200 {object} string "HTML"
+// @Router /api/repos [get]
+func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+		c.Writer.Flush()
+		c.Writer.WriteString("<pre>\n")
+		if len(localRepos) == 0 {
+			c.Writer.WriteString("<a href=\"-/\">default</a>\n")
+		}
+		for publishPrefix := range localRepos {
+			c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
+		}
+		c.Writer.WriteString("</pre>")
+		c.Writer.Flush()
+	}
+}
+
+// @Summary Serve Packages
+// @Description If ServeInAPIMode is enabled in aptly config,
+// @Description this api serves a specified package from storage
+// @Tags Repos
+// @Param storage path string true "Storage"
+// @Param pkgPath path string true "Package Path" allowReserved=true
+// @Produce json
+// @Success 200 ""
+// @Router /api/{storage}/{pkgPath} [get]
+func reposServeInAPIMode(c *gin.Context) {
+	pkgpath := c.Param("pkgPath")
+
+	storage := c.Param("storage")
+	if storage == "-" {
+		storage = ""
+	} else {
+		storage = "filesystem:" + storage
+	}
+
+	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	c.FileFromFS(pkgpath, http.Dir(publicPath))
+}
+
+// @Summary List Repositories
+// @Description **Get list of available repos**
+// @Description Each repo is returned as in “show” API.
+// @Tags Repos
+// @Produce  json
+// @Success 200 {array} deb.LocalRepo
+// @Router /api/repos [get]
 func apiReposList(c *gin.Context) {
 	result := []*deb.LocalRepo{}
 
@@ -31,14 +84,39 @@ func apiReposList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// POST /api/repos
+type repoCreateParams struct {
+	// Name of repository to create
+	Name string `binding:"required"  json:"Name"                 example:"repo1"`
+	// Text describing the repository (optional)
+	Comment string `                 json:"Comment"              example:"this is a repo"`
+	// Default distribution when publishing from this local repo
+	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
+	// Default component when publishing from this local repo
+	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
+	// Snapshot name to create repoitory from (optional)
+	FromSnapshot string `            json:"FromSnapshot"         example:""`
+}
+
+// @Summary Create Repository
+// @Description **Create a local repository**
+// @Description
+// @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot.
+// @Description
+// @Description ```
+// @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos
+// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
+// @Description ```
+// @Tags Repos
+// @Produce  json
+// @Consume  json
+// @Param request body repoCreateParams true "Parameters"
+// @Success 201 {object} deb.LocalRepo
+// @Failure 404 {object} Error "Source snapshot not found"
+// @Failure 409 {object} Error "Local repo already exists"
+// @Failure 500 {object} Error "Internal error"
+// @Router /api/repos [post]
 func apiReposCreate(c *gin.Context) {
-	var b struct {
-		Name                string `binding:"required"`
-		Comment             string
-		DefaultDistribution string
-		DefaultComponent    string
-	}
+	var b repoCreateParams
 
 	if c.Bind(&b) != nil {
 		return
@@ -49,25 +127,65 @@ func apiReposCreate(c *gin.Context) {
 	repo.DefaultDistribution = b.DefaultDistribution
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.LocalRepoCollection()
-	err := collection.Add(repo)
-	if err != nil {
-		c.AbortWithError(400, err)
+
+	if b.FromSnapshot != "" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := collectionFactory.SnapshotCollection()
+
+		snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+		if err != nil {
+			AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(snapshot)
+		if err != nil {
+			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
+			return
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+
+	if _, err := localRepoCollection.ByName(b.Name); err == nil {
+		AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
 		return
 	}
 
-	c.JSON(201, repo)
-}
-
-// PUT /api/repos/:name
-func apiReposEdit(c *gin.Context) {
-	var b struct {
-		Name                *string
-		Comment             *string
-		DefaultDistribution *string
-		DefaultComponent    *string
+	err := localRepoCollection.Add(repo)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
 	}
 
+	c.JSON(http.StatusCreated, repo)
+}
+
+type reposEditParams struct {
+	// Name of repository to modify
+	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
+	// Change Comment of repository
+	Comment *string `                    json:"Comment"              example:"example repo"`
+	// Change Default Distribution for publishing
+	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
+	// Change Devault Component for publishing
+	DefaultComponent *string `        json:"DefaultComponent"     example:""`
+}
+
+// @Summary Update Repository
+// @Description **Update local repository meta information**
+// @Tags Repos
+// @Produce json
+// @Param request body reposEditParams true "Parameters"
+// @Success 200 {object} deb.LocalRepo "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/repos/{name} [put]
+func apiReposEdit(c *gin.Context) {
+	var b reposEditParams
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -77,7 +195,7 @@ func apiReposEdit(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -85,7 +203,7 @@ func apiReposEdit(c *gin.Context) {
 		_, err := collection.ByName(*b.Name)
 		if err == nil {
 			// already exists
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 		repo.Name = *b.Name
@@ -102,7 +220,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -110,20 +228,39 @@ func apiReposEdit(c *gin.Context) {
 }
 
 // GET /api/repos/:name
+// @Summary Get Repository Info
+// @Description Returns basic information about local repository.
+// @Tags Repos
+// @Produce  json
+// @Param name path string true "Repository name"
+// @Success 200 {object} deb.LocalRepo
+// @Failure 404 {object} Error "Repository not found"
+// @Router /api/repos/{name} [get]
 func apiReposShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, repo)
 }
 
-// DELETE /api/repos/:name
+// @Summary Delete Repository
+// @Description Drop/delete a repo
+// @Description Cannot drop repos that are published.
+// @Description Needs force=1 to drop repos used as source by other repos.
+// @Tags Repos
+// @Produce json
+// @Param _async query bool false "Run in background and return task object"
+// @Param force query int false "force: 1 to enable"
+// @Success 200 {object} task.ProcessReturnValue "Repo object"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 404 {object} Error "Repo Conflict"
+// @Router /api/repos/{name} [delete]
 func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	name := c.Params.ByName("name")
@@ -135,13 +272,13 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
@@ -158,31 +295,54 @@ func apiReposDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/repos/:name/packages
+// @Summary List Repo Packages
+// @Description **Return a list of packages present in the repo**
+// @Description
+// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/repos/aptly-repo/packages
+// @Description ["Pi386 aptly 0.8 966561016b44ed80"]
+// @Description ```
+// @Tags Repos
+// @Produce json
+// @Param name path string true "Snapshot to search"
+// @Param q query string true "Package query (e.g Name%20(~%20matlab))"
+// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
+// @Param format query string true "Set to 'details' to return extra info about each package"
+// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
+// @Success 200 {object} string "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 404 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [get]
 func apiReposPackagesShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	showPackages(c, repo.RefList(), collectionFactory)
 }
 
+type reposPackagesAddDeleteParams struct {
+	// Package Refs
+	PackageRefs []string `binding:"required"     json:"PackageRefs"                 example:""`
+}
+
 // Handler for both add and delete
 func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
-	var b struct {
-		PackageRefs []string
-	}
+	var b reposPackagesAddDeleteParams
 
 	if c.Bind(&b) != nil {
 		return
@@ -193,18 +353,18 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
-		return
-	}
-
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
-	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
 		out.Printf("Loading packages...\n")
 		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
 		if err != nil {
@@ -239,7 +399,21 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	})
 }
 
-// POST /repos/:name/packages
+// @Summary Add Packages by Key
+// @Description **Add packages to local repository by package keys.**
+// @Description
+// @Description Any package can be added that is present in the aptly database (from any mirror, snapshot, local repository). This API combined with package list (search) APIs allows one to implement importing, copying, moving packages around.
+// @Description
+// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
+// @Tags Repos
+// @Produce json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 400 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [post]
 func apiReposPackagesAdd(c *gin.Context) {
 	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
 		out.Printf("Adding package %s\n", p.Name)
@@ -247,7 +421,19 @@ func apiReposPackagesAdd(c *gin.Context) {
 	})
 }
 
-// DELETE /repos/:name/packages
+// @Summary Delete Packages by Key
+// @Description **Remove packages from local repository by package keys.**
+// @Description
+// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
+// @Tags Repos
+// @Produce json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 400 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [delete]
 func apiReposPackagesDelete(c *gin.Context) {
 	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
 		out.Printf("Removing package %s\n", p.Name)
@@ -256,13 +442,43 @@ func apiReposPackagesDelete(c *gin.Context) {
 	})
 }
 
-// POST /repos/:name/file/:dir/:file
+// @Summary Add Uploaded File
+// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
+// @Description Adding same package to local repository is not an error.
+// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string false "Filename (optional)"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {string} string "OK"
+// @Failure 400 {object} Error "wrong file"
+// @Failure 404 {object} Error "Repository not found"
+// @Failure 500 {object} Error "Error adding files"
+// @Router /api/repos/{name}/file/{dir}/{file} [post]
 func apiReposPackageFromFile(c *gin.Context) {
 	// redirect all work to dir method
 	apiReposPackageFromDir(c)
 }
 
-// POST /repos/:name/file/:dir
+// @Summary Add Uploaded Directory
+// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
+// @Description Adding same package to local repository is not an error.
+// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory to add"
+// @Consume  json
+// @Param noRemove query string false "when value is set to 1, don’t remove any files"
+// @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce  json
+// @Success 200 {string} string "OK"
+// @Failure 400 {object} Error "wrong file"
+// @Failure 404 {object} Error "Repository not found"
+// @Failure 500 {object} Error "Error adding files"
+// @Router /api/repos/{name}/file/{dir} [post]
 func apiReposPackageFromDir(c *gin.Context) {
 	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
 	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
@@ -271,10 +487,10 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	dirParam := c.Params.ByName("dir")
-	fileParam := c.Params.ByName("file")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -284,13 +500,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
-		return
-	}
-
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -306,7 +516,12 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
 		verifier := context.GetVerifier()
 
 		var (
@@ -382,13 +597,208 @@ func apiReposPackageFromDir(c *gin.Context) {
 	})
 }
 
-// POST /repos/:name/include/:dir/:file
+type reposCopyPackageParams struct {
+	// Copy also dependencies
+	WithDeps bool `json:"with-deps,omitempty"`
+	// Do not perform operations
+	DryRun bool `json:"dry-run,omitempty"`
+}
+
+// @Summary Copy Package
+// @Description Copies a package from a source to destination repository
+// @Tags Repos
+// @Produce json
+// @Param name path string true "Source repo"
+// @Param src path string true "Destination repo"
+// @Param file path string true "File/packages to copy"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} task.ProcessReturnValue "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 422 {object} Error "Unprocessable Entity"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/copy/{src}/{file} [post]
+func apiReposCopyPackage(c *gin.Context) {
+	dstRepoName := c.Params.ByName("name")
+	srcRepoName := c.Params.ByName("src")
+	fileName := c.Params.ByName("file")
+
+	jsonBody := reposCopyPackageParams{
+		WithDeps: false,
+		DryRun:   false,
+	}
+
+	err := c.Bind(&jsonBody)
+	if err != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
+		return
+	}
+
+	var srcRepo *deb.LocalRepo
+	srcRepo, err = collectionFactory.LocalRepoCollection().ByName(srcRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
+		return
+	}
+
+	if srcRepo.UUID == dstRepo.UUID {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest and source are identical"))
+		return
+	}
+
+	taskName := fmt.Sprintf("Copy packages from repo %s to repo %s", srcRepoName, dstRepoName)
+	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
+		}
+
+		srcRefList := srcRepo.RefList()
+
+		reporter := &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
+		}
+
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
+		}
+
+		srcList.PrepareIndex()
+
+		var architecturesList []string
+
+		if jsonBody.WithDeps {
+			dstList.PrepareIndex()
+
+			// Calculate architectures
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = dstList.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			}
+		}
+
+		// srcList.Filter|FilterWithProgress only accept query list
+		queries := make([]deb.PackageQuery, 1)
+		queries[0], err = query.Parse(fileName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
+		}
+
+		toProcess, err := srcList.Filter(deb.FilterOptions{
+			Queries:           queries,
+			WithDependencies:  jsonBody.WithDeps,
+			Source:            dstList,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+			Progress:          context.Progress(),
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
+		}
+
+		if toProcess.Len() == 0 {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("no package found for filter: '%s'", fileName)
+		}
+
+		err = toProcess.ForEach(func(p *deb.Package) error {
+			err = dstList.Add(p)
+			if err != nil {
+				return err
+			}
+
+			name := fmt.Sprintf("added %s-%s(%s)", p.Name, p.Version, p.Architecture)
+			reporter.AddedLines = append(reporter.AddedLines, name)
+			return nil
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("error processing dest add: %s", err)
+		}
+
+		if jsonBody.DryRun {
+			reporter.Warning("Changes not saved, as dry run has been requested")
+		} else {
+			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
+
+			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report": reporter,
+		}}, nil
+	})
+}
+
+// @Summary Include File from Directory
+// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
+// @Tags Repos
+// @Produce json
+// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
+// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
+// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
+// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/repos/{name}/include/{dir}/{file} [post]
 func apiReposIncludePackageFromFile(c *gin.Context) {
 	// redirect all work to dir method
 	apiReposIncludePackageFromDir(c)
 }
 
-// POST /repos/:name/include/:dir
+type reposIncludePackageFromDirReport struct {
+	Warnings []string
+	Added    []string
+	Deleted  []string
+}
+
+type reposIncludePackageFromDirResponse struct {
+	Report      reposIncludePackageFromDirReport
+	FailedFiles []string
+}
+
+// @Summary Include Directory
+// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
+// @Tags Repos
+// @Produce json
+// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
+// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
+// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
+// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} reposIncludePackageFromDirResponse "Response"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/repos/{name}/include/{dir} [post]
 func apiReposIncludePackageFromDir(c *gin.Context) {
 	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
 	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
@@ -404,10 +814,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	var sources []string
 	var taskName string
-	dirParam := c.Params.ByName("dir")
-	fileParam := c.Params.ByName("file")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -421,7 +831,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("error parsing repo template: %s", err))
 		return
 	}
 
@@ -432,7 +842,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		// repo template string is simple text so only use resource key of specific repository
 		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
 		if err != nil {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
@@ -440,7 +850,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	}
 	resources = append(resources, sources...)
 
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()
@@ -490,6 +900,5 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 			"Report":      reporter,
 			"FailedFiles": failedFiles,
 		}}, nil
-
 	})
 }

api/router.go

@@ -2,36 +2,91 @@ package api
 
 import (
 	"net/http"
+	"os"
+	"sync/atomic"
 
+	"github.com/aptly-dev/aptly/aptly"
 	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog/log"
+
+	"github.com/aptly-dev/aptly/docs"
+	swaggerFiles "github.com/swaggo/files"
+	ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
+		countPackagesByRepos()
 		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
 	}
 }
 
+func redirectSwagger(c *gin.Context) {
+	if c.Request.URL.Path == "/docs/index.html" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs/" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	c.Next()
+}
+
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
-	context = c
-
-	router := gin.Default()
-	router.UseRawPath = true
-	router.Use(gin.ErrorLogger())
-
-	if c.Config().EnableMetricsEndpoint {
-		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+	if aptly.EnableDebug {
+		gin.SetMode(gin.DebugMode)
+	} else {
+		gin.SetMode(gin.ReleaseMode)
 	}
 
+	router := gin.New()
+	context = c
+
+	router.UseRawPath = true
+
+	if c.Config().LogFormat == "json" {
+		c.StructuredLogging(true)
+		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
+		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
+		router.Use(JSONLogger())
+	} else {
+		c.StructuredLogging(false)
+		utils.SetupDefaultLogger(c.Config().LogLevel)
+		router.Use(gin.Logger())
+	}
+
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	if c.Config().EnableSwaggerEndpoint {
+		router.GET("docs.html", func(c *gin.Context) {
+			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+		})
+		router.Use(redirectSwagger)
+		url := ginSwagger.URL("/docs/doc.json")
+		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	}
+
+	if c.Config().EnableMetricsEndpoint {
+		MetricsCollectorRegistrar.Register(router)
+	}
+
+	if c.Config().ServeInAPIMode {
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
+		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
+	}
+
+	api := router.Group("/api")
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
@@ -40,7 +95,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 		go acquireDatabase()
 
-		router.Use(func(c *gin.Context) {
+		api.Use(func(c *gin.Context) {
 			var err error
 
 			errCh := make(chan error)
@@ -48,7 +103,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			err = <-errCh
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -56,7 +111,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 				dbRequests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.AbortWithError(500, err)
+					AbortWithJSONError(c, 500, err)
 				}
 			}()
 
@@ -64,99 +119,119 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		})
 	}
 
-	root := router.Group("/api")
-
 	{
 		if c.Config().EnableMetricsEndpoint {
-			root.GET("/metrics", apiMetricsGet())
+			api.GET("/metrics", apiMetricsGet())
 		}
-		root.GET("/version", apiVersion)
+		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
+
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		api.GET("/ready", apiReady(isReady))
+		api.GET("/healthy", apiHealthy)
 	}
 
 	{
-		root.GET("/repos", apiReposList)
-		root.POST("/repos", apiReposCreate)
-		root.GET("/repos/:name", apiReposShow)
-		root.PUT("/repos/:name", apiReposEdit)
-		root.DELETE("/repos/:name", apiReposDrop)
+		api.GET("/repos", apiReposList)
+		api.POST("/repos", apiReposCreate)
+		api.GET("/repos/:name", apiReposShow)
+		api.PUT("/repos/:name", apiReposEdit)
+		api.DELETE("/repos/:name", apiReposDrop)
 
-		root.GET("/repos/:name/packages", apiReposPackagesShow)
-		root.POST("/repos/:name/packages", apiReposPackagesAdd)
-		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+		api.GET("/repos/:name/packages", apiReposPackagesShow)
+		api.POST("/repos/:name/packages", apiReposPackagesAdd)
+		api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
 
-		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
-		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
 
-		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
-		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+		api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
 
-		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+		api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 
 	{
-		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+		api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
 	{
-		root.GET("/mirrors", apiMirrorsList)
-		root.GET("/mirrors/:name", apiMirrorsShow)
-		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
-		root.POST("/mirrors", apiMirrorsCreate)
-		root.PUT("/mirrors/:name", apiMirrorsUpdate)
-		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+		api.GET("/mirrors", apiMirrorsList)
+		api.GET("/mirrors/:name", apiMirrorsShow)
+		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		api.POST("/mirrors", apiMirrorsCreate)
+		api.PUT("/mirrors/:name", apiMirrorsUpdate)
+		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
-		root.POST("/gpg/key", apiGPGAddKey)
+		api.POST("/gpg/key", apiGPGAddKey)
 	}
 
 	{
-		root.GET("/files", apiFilesListDirs)
-		root.POST("/files/:dir", apiFilesUpload)
-		root.GET("/files/:dir", apiFilesListFiles)
-		root.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+		api.GET("/s3", apiS3List)
 	}
 
 	{
-		root.GET("/publish", apiPublishList)
-		root.POST("/publish", apiPublishRepoOrSnapshot)
-		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
-		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
-		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.GET("/files", apiFilesListDirs)
+		api.POST("/files/:dir", apiFilesUpload)
+		api.GET("/files/:dir", apiFilesListFiles)
+		api.DELETE("/files/:dir", apiFilesDeleteDir)
+		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
 	}
 
 	{
-		root.GET("/snapshots", apiSnapshotsList)
-		root.POST("/snapshots", apiSnapshotsCreate)
-		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
-		root.GET("/snapshots/:name", apiSnapshotsShow)
-		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
-		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
-		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.GET("/publish", apiPublishList)
+		api.GET("/publish/:prefix/:distribution", apiPublishShow)
+		api.POST("/publish", apiPublishRepoOrSnapshot)
+		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource)
+		api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges)
+		api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources)
+		api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges)
+		api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource)
+		api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource)
+		api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate)
 	}
 
 	{
-		root.GET("/packages/:key", apiPackagesShow)
+		api.GET("/snapshots", apiSnapshotsList)
+		api.POST("/snapshots", apiSnapshotsCreate)
+		api.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.GET("/snapshots/:name", apiSnapshotsShow)
+		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
 	}
 
 	{
-		root.GET("/graph.:ext", apiGraph)
+		api.GET("/packages/:key", apiPackagesShow)
+		api.GET("/packages", apiPackages)
+	}
+
+	{
+		api.GET("/graph.:ext", apiGraph)
 	}
 	{
-		root.POST("/db/cleanup", apiDbCleanup)
+		api.POST("/db/cleanup", apiDbCleanup)
 	}
 	{
-		root.GET("/tasks", apiTasksList)
-		root.POST("/tasks-clear", apiTasksClear)
-		root.GET("/tasks-wait", apiTasksWait)
-		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
-		root.GET("/tasks/:id/output", apiTasksOutputShow)
-		root.GET("/tasks/:id/detail", apiTasksDetailShow)
-		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
-		root.GET("/tasks/:id", apiTasksShow)
-		root.DELETE("/tasks/:id", apiTasksDelete)
-		root.POST("/tasks-dummy", apiTasksDummy)
+		api.GET("/tasks", apiTasksList)
+		api.POST("/tasks-clear", apiTasksClear)
+		api.GET("/tasks-wait", apiTasksWait)
+		api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		api.GET("/tasks/:id/output", apiTasksOutputShow)
+		api.GET("/tasks/:id/detail", apiTasksDetailShow)
+		api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		api.GET("/tasks/:id", apiTasksShow)
+		api.DELETE("/tasks/:id", apiTasksDelete)
 	}
 
 	return router

api/s3.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary S3 buckets
+// @Description **Get list of S3 buckets**
+// @Description
+// @Description List configured S3 buckets.
+// @Tags Status
+// @Produce json
+// @Success 200 {array} string "List of S3 buckets"
+// @Router /api/s3 [get]
+func apiS3List(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().S3PublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/snapshot.go

@@ -3,15 +3,25 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/snapshots
+// @Summary List Snapshots
+// @Description **Get list of snapshots**
+// @Description
+// @Description Each snapshot is returned as in “show” API.
+// @Tags Snapshots
+// @Produce  json
+// @Success 200 {array} deb.Snapshot
+// @Router /api/snapshots [get]
 func apiSnapshotsList(c *gin.Context) {
 	SortMethodString := c.Request.URL.Query().Get("sort")
 
@@ -31,19 +41,34 @@ func apiSnapshotsList(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// POST /api/mirrors/:name/snapshots/
+type snapshotsCreateFromMirrorParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"     json:"Name"                 example:"snap1"`
+	// Description of snapshot
+	Description string `                json:"Description"`
+}
+
+// @Summary Snapshot Mirror
+// @Description **Create a snapshot of a mirror**
+// @Tags Snapshots
+// @Produce json
+// @Param request body snapshotsCreateFromMirrorParams true "Parameters"
+// @Param name path string true "Mirror name"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 201 {object} deb.Snapshot "Created Snapshot"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Mirror Not Found"
+// @Failure 409 {object} Error "Conflicting snapshot"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/mirrors/{name}/snapshots [post]
 func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	var (
 		err      error
 		repo     *deb.RemoteRepo
 		snapshot *deb.Snapshot
+		b        snapshotsCreateFromMirrorParams
 	)
 
-	var b struct {
-		Name        string `binding:"required"`
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -55,14 +80,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
@@ -90,20 +115,37 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	})
 }
 
-// POST /api/snapshots
+type snapshotsCreateParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"  json:"Name"                 example:"snap2"`
+	// Description of snapshot
+	Description string `             json:"Description"`
+	// List of source snapshots
+	SourceSnapshots []string `       json:"SourceSnapshots"      example:"snap1"`
+	// List of package refs
+	PackageRefs []string `           json:"PackageRefs"          example:""`
+}
+
+// @Summary Snapshot Packages
+// @Description **Create a snapshot from package refs**
+// @Description
+// @Description Refs can be obtained from snapshots, local repos, or mirrors
+// @Tags Snapshots
+// @Param request body snapshotsCreateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 201 {object} deb.Snapshot "Created snapshot"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Source snapshot or package refs not found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots [post]
 func apiSnapshotsCreate(c *gin.Context) {
 	var (
 		err      error
 		snapshot *deb.Snapshot
+		b        snapshotsCreateParams
 	)
 
-	var b struct {
-		Name            string `binding:"required"`
-		Description     string
-		SourceSnapshots []string
-		PackageRefs     []string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -123,20 +165,21 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.AbortWithError(404, err)
-			return
-		}
-
-		err = snapshotCollection.LoadComplete(sources[i])
-		if err != nil {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
 		resources = append(resources, string(sources[i].ResourceKey()))
 	}
 
-	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		for i := range sources {
+			err = snapshotCollection.LoadComplete(sources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+		}
+
 		list := deb.NewPackageList()
 
 		// verify package refs and build package list
@@ -160,23 +203,39 @@ func apiSnapshotsCreate(c *gin.Context) {
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
-		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
 	})
 }
 
-// POST /api/repos/:name/snapshots
+type snapshotsCreateFromRepositoryParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"               json:"Name"                 example:"snap1"`
+	// Description of snapshot
+	Description string `                          json:"Description"`
+}
+
+// @Summary Snapshot Repository
+// @Description **Create a snapshot of a repository by name**
+// @Tags Snapshots
+// @Param name path string true "Repository name"
+// @Consume json
+// @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
+// @Param name path string true "Name of the snapshot"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 201 {object} deb.Snapshot "Created snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Failure 404 {object} Error "Repo Not Found"
+// @Router /api/repos/{name}/snapshots [post]
 func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	var (
 		err      error
 		repo     *deb.LocalRepo
 		snapshot *deb.Snapshot
+		b        snapshotsCreateFromRepositoryParams
 	)
 
-	var b struct {
-		Name        string `binding:"required"`
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -188,14 +247,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -218,18 +277,32 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	})
 }
 
-// PUT /api/snapshots/:name
+type snapshotsUpdateParams struct {
+	// Change Name of snapshot
+	Name string `       json:"Name"  example:"snap2"`
+	// Change Description of snapshot
+	Description string `json:"Description"`
+}
+
+// @Summary Update Snapshot
+// @Description **Update snapshot metadata (Name, Description)**
+// @Tags Snapshots
+// @Param request body snapshotsUpdateParams true "Parameters"
+// @Param name path string true "Snapshot name"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} deb.Snapshot "Updated snapshot object"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 409 {object} Error "Conflicting snapshot"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [put]
 func apiSnapshotsUpdate(c *gin.Context) {
 	var (
 		err      error
 		snapshot *deb.Snapshot
+		b        snapshotsUpdateParams
 	)
 
-	var b struct {
-		Name        string
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -240,13 +313,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
 	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		_, err := collection.ByName(b.Name)
 		if err == nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
@@ -268,27 +341,48 @@ func apiSnapshotsUpdate(c *gin.Context) {
 	})
 }
 
-// GET /api/snapshots/:name
+// @Summary Get Snapshot Info
+// @Description **Query detailed information about a snapshot by name**
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot"
+// @Produce json
+// @Success 200 {object} deb.Snapshot "msg"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [get]
 func apiSnapshotsShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, snapshot)
 }
 
-// DELETE /api/snapshots/:name
+// @Summary Delete Snapshot
+// @Description **Delete snapshot by name**
+// @Description Cannot drop snapshots that are published.
+// @Description Needs force=1 to drop snapshots used as source by other snapshots.
+// @Tags Snapshots
+// @Param name path string true "Snapshot name"
+// @Param force query string false "Force operation"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 ""
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 409 {object} Error "Snapshot in use"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [delete]
 func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
@@ -299,13 +393,13 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
@@ -327,7 +421,19 @@ func apiSnapshotsDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/snapshots/:name/diff/:withSnapshot
+// @Summary Snapshot diff
+// @Description **Return the diff between two snapshots (name & withSnapshot)**
+// @Description Provide `onlyMatching=1` to return only packages present in both snapshots.
+// @Description Otherwise, returns a `left` and `right` result providing packages only in the first and second snapshots
+// @Tags Snapshots
+// @Produce json
+// @Param name path string true "Snapshot name"
+// @Param withSnapshot path string true "Snapshot name to diff against"
+// @Param onlyMatching query string false "Only return packages present in both snapshots"
+// @Success 200 {array} deb.PackageDiff "Package Diff"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name}/diff/{withSnapshot} [get]
 func apiSnapshotsDiff(c *gin.Context) {
 	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
 
@@ -336,32 +442,32 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -378,22 +484,334 @@ func apiSnapshotsDiff(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// GET /api/snapshots/:name/packages
+// @Summary List Snapshot Packages
+// @Description **List all packages in snapshot or perform search on snapshot contents and return results**
+// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
+// @Tags Snapshots
+// @Produce json
+// @Param name path string true "Snapshot to search"
+// @Param q query string false "Package query (e.g Name%20(~%20matlab))"
+// @Param withDeps query string false "Set to 1 to include dependencies when evaluating package query"
+// @Param format query string false "Set to 'details' to return extra info about each package"
+// @Param maximumVersion query string false "Set to 1 to only return the highest version for each package name"
+// @Success 200 {array} string "Package info"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name}/packages [get]
 func apiSnapshotsSearchPackages(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	showPackages(c, snapshot.RefList(), collectionFactory)
 }
+
+type snapshotsMergeParams struct {
+	// List of snapshot names to be merged
+	Sources []string `binding:"required" json:"Sources"     example:"snapshot1"`
+}
+
+// @Summary Snapshot Merge
+// @Description **Merge several source snapshots into a new snapshot**
+// @Description
+// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
+// @Description
+// @Description If only one snapshot is specified, merge copies source into destination.
+// @Tags Snapshots
+// @Consume json
+// @Produce json
+// @Param name path string true "Name of the snapshot to be created"
+// @Param latest query int false "merge only the latest version of each package"
+// @Param no-remove query int false "all versions of packages are preserved during merge"
+// @Param request body snapshotsMergeParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 201 {object} deb.Snapshot "Resulting snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/merge [post]
+func apiSnapshotsMerge(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+		body     snapshotsMergeParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if c.Bind(&body) != nil {
+		return
+	}
+
+	if len(body.Sources) < 1 {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
+		return
+	}
+
+	latest := c.Request.URL.Query().Get("latest") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+	overrideMatching := !latest && !noRemove
+
+	if noRemove && latest {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("no-remove and latest are mutually exclusive"))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	sources := make([]*deb.Snapshot, len(body.Sources))
+	resources := make([]string, len(sources))
+	for i := range body.Sources {
+		sources[i], err = snapshotCollection.ByName(body.Sources[i])
+		if err != nil {
+			AbortWithJSONError(c, http.StatusNotFound, err)
+			return
+		}
+
+		resources[i] = string(sources[i].ResourceKey())
+	}
+
+	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = snapshotCollection.LoadComplete(sources[0])
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		result := sources[0].RefList()
+		for i := 1; i < len(sources); i++ {
+			err = snapshotCollection.LoadComplete(sources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+		}
+
+		if latest {
+			result.FilterLatestRefs()
+		}
+
+		sourceDescription := make([]string, len(sources))
+		for i, s := range sources {
+			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
+		}
+
+		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+type snapshotsPullParams struct {
+	// Source name to be searched for packages and dependencies
+	Source string `binding:"required"      json:"Source"            example:"source-snapshot"`
+	// Name of the snapshot to be created
+	Destination string `binding:"required" json:"Destination"       example:"idestination-snapshot"`
+	// List of package queries (i.e. name of package to be pulled from `Source`)
+	Queries []string `binding:"required"   json:"Queries"           example:"xserver-xorg"`
+	// List of architectures (optional)
+	Architectures []string `               json:"Architectures"     example:"amd64, armhf"`
+}
+
+// @Summary Snapshot Pull
+// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
+// @Description
+// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
+// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
+// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
+// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
+// @Description
+// @Description With flag `no-remove` both package versions would stay in the snapshot.
+// @Description
+// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
+// @Tags Snapshots
+// @Param request body snapshotsPullParams true "Parameters"
+// @Param name path string true "Name of the snapshot to be created"
+// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
+// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
+// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
+// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Consume json
+// @Produce json
+// @Success 200 {object} deb.Snapshot "Resulting Snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/pull [post]
+func apiSnapshotsPull(c *gin.Context) {
+	var (
+		err                 error
+		destinationSnapshot *deb.Snapshot
+		body                snapshotsPullParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if err = c.BindJSON(&body); err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, err)
+		return
+	}
+
+	allMatches := c.Request.URL.Query().Get("all-matches") == "1"
+	dryRun := c.Request.URL.Query().Get("dry-run") == "1"
+	noDeps := c.Request.URL.Query().Get("no-deps") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name> snapshot
+	toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+
+	// Load <Source> snapshot
+	sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+
+	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// convert snapshots to package list
+		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		toPackageList.PrepareIndex()
+		sourcePackageList.PrepareIndex()
+
+		var architecturesList []string
+
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = toPackageList.Architectures(false)
+		}
+
+		architecturesList = append(architecturesList, body.Architectures...)
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+		var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+		for _, arch := range architecturesList {
+			archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+		}
+
+		queries := make([]deb.PackageQuery, len(body.Queries))
+		for i, q := range body.Queries {
+			queries[i], err = query.Parse(q)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// Add architecture filter
+			queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+		}
+
+		// Filter with dependencies as requested
+		destinationPackageList, err := sourcePackageList.Filter(deb.FilterOptions{
+			Queries:           queries,
+			WithDependencies:  !noDeps,
+			Source:            toPackageList,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+			Progress:          context.Progress(),
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		destinationPackageList.PrepareIndex()
+
+		removedPackages := []string{}
+		addedPackages := []string{}
+		alreadySeen := map[string]bool{}
+
+		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+			key := pkg.Architecture + "_" + pkg.Name
+			_, seen := alreadySeen[key]
+
+			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+			if !noRemove && !seen {
+				// Remove all packages with the same name and architecture
+				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
+				for _, p := range packageSearchResults {
+					toPackageList.Remove(p)
+					removedPackages = append(removedPackages, p.String())
+				}
+			}
+
+			// If !allMatches, add only first matching name-arch package
+			if !seen || allMatches {
+				toPackageList.Add(pkg)
+				addedPackages = append(addedPackages, pkg.String())
+			}
+
+			alreadySeen[key] = true
+
+			return nil
+		})
+		alreadySeen = nil
+
+		if dryRun {
+			response := struct {
+				AddedPackages   []string `json:"added_packages"`
+				RemovedPackages []string `json:"removed_packages"`
+			}{
+				AddedPackages:   addedPackages,
+				RemovedPackages: removedPackages,
+			}
+
+			return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
+		}
+
+		// Create <destination> snapshot
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
+	})
+}

api/storage.go

@@ -0,0 +1,45 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Description
+// @Description Units in MiB.
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

api/task.go

@@ -1,155 +1,203 @@
 package api
 
 import (
-	"fmt"
-	"net/http"
 	"strconv"
 
-	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
-// GET /tasks
+// @Summary List Tasks
+// @Description **Get list of available tasks. Each task is returned as in “show” API**
+// @Tags Tasks
+// @Produce json
+// @Success 200 {array} task.Task
+// @Router /api/tasks [get]
 func apiTasksList(c *gin.Context) {
 	list := context.TaskList()
 	c.JSON(200, list.GetTasks())
 }
 
-// POST /tasks-clear
+// @Summary Clear Tasks
+// @Description **Removes finished and failed tasks from internal task list**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-clear [post]
 func apiTasksClear(c *gin.Context) {
 	list := context.TaskList()
 	list.Clear()
 	c.JSON(200, gin.H{})
 }
 
-// GET /tasks-wait
+// @Summary Wait for all Tasks
+// @Description **Waits for and returns when all running tasks are complete**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-wait [get]
 func apiTasksWait(c *gin.Context) {
 	list := context.TaskList()
 	list.Wait()
 	c.JSON(200, gin.H{})
 }
 
-// GET /tasks/:id/wait
+// @Summary Wait for Task
+// @Description **Waits for and returns when given Task ID is complete**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 400 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/wait [get]
 func apiTasksWaitForTaskByID(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	task, err := list.WaitForTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, task)
 }
 
-// GET /tasks/:id
+// @Summary Get Task Info
+// @Description **Return task information for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id} [get]
 func apiTasksShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var task task.Task
 	task, err = list.GetTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, task)
 }
 
-// GET /tasks/:id/output
+// @Summary Get Task Output
+// @Description **Return task output for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task output"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/output [get]
 func apiTasksOutputShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var output string
 	output, err = list.GetTaskOutputByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, output)
 }
 
-// GET /tasks/:id/detail
+// @Summary Get Task Details
+// @Description **Return task detail for a given ID**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task detail"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/detail [get]
 func apiTasksDetailShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var detail interface{}
 	detail, err = list.GetTaskDetailByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, detail)
 }
 
-// GET /tasks/:id/return_value
+// @Summary Get Task Return Value
+// @Description **Return task return value (status code) by given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/tasks/{id}/return_value [get]
 func apiTasksReturnValueShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err := list.GetTaskReturnValueByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, output)
 }
 
-// DELETE /tasks/:id
+// @Summary Delete Task
+// @Description **Delete completed task by given ID. Does not stop task execution**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 400 {object} Error "Task in progress or not found"
+// @Router /api/tasks/{id} [delete]
 func apiTasksDelete(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var delTask task.Task
 	delTask, err = list.DeleteTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, delTask)
 }
-
-// POST /tasks-dummy
-func apiTasksDummy(c *gin.Context) {
-	resources := []string{"dummy"}
-	taskName := fmt.Sprintf("Dummy task")
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		out.Printf("Dummy task started\n")
-		detail.Store([]int{1, 2, 3})
-		out.Printf("Dummy task finished\n")
-		return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
-	})
-}

api/task_test.go

@@ -1,75 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/aptly-dev/aptly/task"
-
-	. "gopkg.in/check.v1"
-)
-
-type TaskSuite struct {
-	ApiSuite
-}
-
-var _ = Suite(&TaskSuite{})
-
-func (s *TaskSuite) TestTasksDummy(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
-	c.Check(response.Code, Equals, 418)
-	c.Check(response.Body.String(), Equals, "[1,2,3]")
-}
-
-func (s *TaskSuite) TestTasksDummyAsync(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	var t task.Task
-	err := json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.Name, Equals, "Dummy task")
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
-	err = json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.State, Equals, task.SUCCEEDED)
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "[1,2,3]")
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
-}
-
-func (s *TaskSuite) TestTaskDelete(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
-	// Give the task time to start
-	time.Sleep(time.Second)
-	response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
-	c.Check(response.Code, Equals, 200)
-}
-
-func (s *TaskSuite) TestTasksClear(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	var t task.Task
-	err := json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.Name, Equals, "Dummy task")
-	response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
-	c.Check(response.Code, Equals, 200)
-	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
-	c.Check(response.Code, Equals, 200)
-	var ts []task.Task
-	err = json.Unmarshal(response.Body.Bytes(), &ts)
-	c.Assert(err, IsNil)
-	c.Check(len(ts), Equals, 1)
-	c.Check(ts[0].State, Equals, task.SUCCEEDED)
-	response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
-	c.Check(response.Code, Equals, 200)
-	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "null")
-}

aptly-api.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository API
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/api/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
-
-[Install]
-WantedBy=multi-user.target

aptly.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository server
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/commands/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
-
-[Install]
-WantedBy=multi-user.target

aptly/conf.go

@@ -0,0 +1,4 @@
+package aptly
+
+// Default aptly.conf (filled in at link time)
+var AptlyConf []byte

aptly/const.go

@@ -0,0 +1,3 @@
+package aptly
+
+var DistributionFocal = "focal"

aptly/interfaces.go

@@ -35,8 +35,8 @@ type PackagePool interface {
 	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
 	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
 	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
-	// Stat returns Unix stat(2) info
-	Stat(path string) (os.FileInfo, error)
+	// Size returns the size of the given file in bytes.
+	Size(path string) (size int64, err error)
 	// Open returns ReadSeekerCloser to access the file
 	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
@@ -47,6 +47,8 @@ type PackagePool interface {
 
 // LocalPackagePool is implemented by PackagePools residing on the same filesystem
 type LocalPackagePool interface {
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
 	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
 	GenerateTempPath(filename string) (string, error)
 	// Link generates hardlink to destination path
@@ -70,7 +72,7 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file

azure/azure.go

@@ -1,2 +1,136 @@
-// Package azure handles publishing to Azure Storage
 package azure
+
+// Package azure handles publishing to Azure Storage
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+func isBlobNotFound(err error) bool {
+	var respErr *azcore.ResponseError
+	if errors.As(err, &respErr) {
+		return respErr.StatusCode == 404 // BlobNotFound
+	}
+	return false
+}
+
+type azContext struct {
+	client    *azblob.Client
+	container string
+	prefix    string
+}
+
+func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
+	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+	}
+
+	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	result := &azContext{
+		client:    serviceClient,
+		container: container,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+func (az *azContext) blobPath(path string) string {
+	return filepath.Join(az.prefix, path)
+}
+
+func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(az.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	ctx := context.Background()
+	maxResults := int32(1)
+	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
+		Prefix:     &prefix,
+		MaxResults: &maxResults,
+		Include:    azblob.ListBlobsInclude{Metadata: true},
+	})
+
+	// Iterate over each page
+	for pager.More() {
+		page, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
+		}
+
+		for _, blob := range page.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, *blob.Name)
+			} else {
+				name := *blob.Name
+				paths = append(paths, name[len(prefix):])
+			}
+			b := *blob
+			md5 := b.Properties.ContentMD5
+			md5s = append(md5s, fmt.Sprintf("%x", md5))
+
+		}
+		if progress != nil {
+			time.Sleep(time.Duration(500) * time.Millisecond)
+			progress.AddBar(1)
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
+	uploadOptions := &azblob.UploadFileOptions{
+		BlockSize:   4 * 1024 * 1024,
+		Concurrency: 8,
+	}
+
+	path := az.blobPath(blobName)
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
+			BlobContentMD5: decodedMD5,
+		}
+	}
+
+	var err error
+	if file, ok := source.(*os.File); ok {
+		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
+	}
+
+	return err
+}
+
+// String
+func (az *azContext) String() string {
+	return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
+}

azure/package_pool.go

@@ -0,0 +1,215 @@
+package azure
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+type PackagePool struct {
+	az *azContext
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates published storage from Azure storage credentials
+func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagePool{az: azctx}, nil
+}
+
+// String
+func (pool *PackagePool) String() string {
+	return pool.az.String()
+}
+
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
+	hash := checksums.SHA256
+	// Use the same path as the file pool, for compat reasons.
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
+}
+
+func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+	targetChecksums, err := checksumStorage.Get(poolPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		if err != nil {
+			if isBlobNotFound(err) {
+				return nil, nil
+			}
+
+			return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
+		}
+
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return targetChecksums, nil
+}
+
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	if progress != nil {
+		progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
+		defer progress.ShutdownBar()
+	}
+
+	paths, _, err := pool.az.internalFilelist("", progress)
+	return paths, err
+}
+
+func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
+	return "", errors.New("Azure package pool does not support legacy paths")
+}
+
+func (pool *PackagePool) Size(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	temp, err := os.CreateTemp("", "blob-download")
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+	}
+	defer os.Remove(temp.Name())
+
+	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+	}
+
+	return temp, nil
+}
+
+func (pool *PackagePool) Remove(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		var err error
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	path := pool.buildPoolPath(basename, checksums)
+	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
+	if err != nil {
+		return "", err
+	} else if targetChecksums != nil {
+		// target already exists
+		*checksums = *targetChecksums
+		return path, nil
+	}
+
+	source, err := os.Open(srcPath)
+	if err != nil {
+		return "", err
+	}
+	defer source.Close()
+
+	err = pool.az.putFile(path, source, checksums.MD5)
+	if err != nil {
+		return "", err
+	}
+
+	if !checksums.Complete() {
+		// need full checksums here
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	err = checksumStorage.Update(path, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	if poolPath == "" {
+		if checksums.SHA256 != "" {
+			poolPath = pool.buildPoolPath(basename, checksums)
+		} else {
+			// No checksums or pool path, so no idea what file to look for.
+			return "", false, nil
+		}
+	}
+
+	size, err := pool.Size(poolPath)
+	if err != nil {
+		return "", false, err
+	} else if size != checksums.Size {
+		return "", false, nil
+	}
+
+	targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
+	if err != nil {
+		return "", false, err
+	} else if targetChecksums == nil {
+		return "", false, nil
+	}
+
+	if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+		checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+		// wrong file?
+		return "", false, nil
+	}
+
+	// fill back checksums
+	*checksums = *targetChecksums
+	return poolPath, true, nil
+}

azure/package_pool_test.go

@@ -0,0 +1,257 @@
+package azure
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type PackagePoolSuite struct {
+	accountName, accountKey, endpoint string
+	pool, prefixedPool                *PackagePool
+	debFile                           string
+	cs                                aptly.ChecksumStorage
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
+	c.Assert(err, IsNil)
+
+	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = files.NewMockChecksumStorage()
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
+	})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	var checksum utils.ChecksumInfo
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	// import as different name
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	// double import, should be ok
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with path-relevant checksums already filled in
+	checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
+	path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	checksum := utils.ChecksumInfo{}
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with checksums missing (that aren't needed to find the path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
+		SHA256: "abc",
+		Size:   checksum.Size,
+	}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksums (that aren't needed to find the path)
+	// and no info in checksum storage
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	checksum = utils.ChecksumInfo{Size: 13455}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestSize(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Size("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}

azure/public.go

@@ -2,27 +2,25 @@ package azure
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
-	"io"
-	"net"
-	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
 // PublishedStorage abstract file system with published files (actually hosted on Azure)
 type PublishedStorage struct {
-	container azblob.ContainerURL
 	prefix    string
-	pathCache map[string]string
+	az        *azContext
+	pathCache map[string]map[string]string
 }
 
 // Check interface
@@ -30,60 +28,23 @@ var (
 	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
 )
 
-func isEmulatorEndpoint(endpoint string) bool {
-	if h, _, err := net.SplitHostPort(endpoint); err == nil {
-		endpoint = h
-	}
-	if endpoint == "localhost" {
-		return true
-	}
-	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
-	// In this case, eliminate the '[' and ']' in the URL.
-	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
-	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
-		endpoint = endpoint[1 : len(endpoint)-1]
-	}
-	return net.ParseIP(endpoint) != nil
-}
-
 // NewPublishedStorage creates published storage from Azure storage credentials
 func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
-	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	if endpoint == "" {
-		endpoint = "blob.core.windows.net"
-	}
-
-	var url *url.URL
-	if isEmulatorEndpoint(endpoint) {
-		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
-	} else {
-		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
-
-	result := &PublishedStorage{
-		container: containerURL,
-		prefix:    prefix,
-	}
-
-	return result, nil
+	return &PublishedStorage{az: azctx}, nil
 }
 
 // String
 func (storage *PublishedStorage) String() string {
-	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+	return storage.az.String()
 }
 
 // MkDir creates directory recursively under public path
-func (storage *PublishedStorage) MkDir(path string) error {
+func (storage *PublishedStorage) MkDir(_ string) error {
 	// no op for Azure
 	return nil
 }
@@ -106,7 +67,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer source.Close()
 
-	err = storage.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(path, source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -114,46 +75,17 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	return err
 }
 
-// putFile uploads file-like object to
-func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
-	path = filepath.Join(storage.prefix, path)
-
-	blob := storage.container.NewBlockBlobURL(path)
-
-	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize: 4 * 1024 * 1024,
-		MaxBuffers: 8,
-	}
-	if len(sourceMD5) > 0 {
-		decodedMD5, err := hex.DecodeString(sourceMD5)
-		if err != nil {
-			return err
-		}
-		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
-			ContentMD5: decodedMD5,
-		}
-	}
-
-	_, err := azblob.UploadStreamToBlockBlob(
-		context.Background(),
-		source,
-		blob,
-		uploadOptions,
-	)
-
-	return err
-}
-
 // RemoveDirs removes directory structure under public path
-func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
+	path = storage.az.blobPath(path)
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
-		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+		blob := filepath.Join(path, filename)
+		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
 		}
@@ -164,8 +96,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
-	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	path = storage.az.blobPath(path)
+	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
 	}
@@ -174,31 +106,38 @@ func (storage *PublishedStorage) Remove(path string) error {
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, fileName)
-	poolPath := filepath.Join(storage.prefix, relPath)
+	relFilePath := filepath.Join(publishedRelPath, fileName)
+	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	poolPath := storage.az.blobPath(prefixRelFilePath)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist("")
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
 		if err != nil {
 			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
 
-		storage.pathCache = make(map[string]string, len(paths))
+		pathCache = make(map[string]string, len(paths))
 
 		for i := range paths {
-			storage.pathCache[paths[i]] = md5s[i]
+			pathCache[paths[i]] = md5s[i]
 		}
+		storage.pathCache[publishedPrefix] = pathCache
 	}
 
-	destinationMD5, exists := storage.pathCache[relPath]
+	destinationMD5, exists := pathCache[relFilePath]
 	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
@@ -221,9 +160,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	}
 	defer source.Close()
 
-	err = storage.putFile(relPath, source, sourceMD5)
+	err = storage.az.putFile(relFilePath, source, sourceMD5)
 	if err == nil {
-		storage.pathCache[relPath] = sourceMD5
+		pathCache[relFilePath] = sourceMD5
 	} else {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}
@@ -231,98 +170,65 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	return err
 }
 
-func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
-	const delimiter = "/"
-	paths = make([]string, 0, 1024)
-	md5s = make([]string, 0, 1024)
-	prefix = filepath.Join(storage.prefix, prefix)
-	if prefix != "" {
-		prefix += delimiter
-	}
-
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		listBlob, err := storage.container.ListBlobsFlatSegment(
-			context.Background(), marker, azblob.ListBlobsSegmentOptions{
-				Prefix:     prefix,
-				MaxResults: 1000,
-				Details:    azblob.BlobListingDetails{Metadata: true}})
-		if err != nil {
-			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
-		}
-
-		marker = listBlob.NextMarker
-
-		for _, blob := range listBlob.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, blob.Name)
-			} else {
-				paths = append(paths, blob.Name[len(prefix):])
-			}
-			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
-		}
-	}
-
-	return paths, md5s, nil
-}
-
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
-	paths, _, err := storage.internalFilelist(prefix)
+	paths, _, err := storage.az.internalFilelist(prefix, nil)
 	return paths, err
 }
 
 // Internal copy or move implementation
-func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
 	const leaseDuration = 30
+	leaseID := uuid.NewString()
 
-	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
-	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
-	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
-	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
-		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	srcBlobClient := containerClient.NewBlobClient(src)
+	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
 	}
-	defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
-	srcBlobLeaseID := leaseResp.LeaseID()
 
-	copyResp, err := dstBlobURL.StartCopyFromURL(
-		context.Background(),
-		srcBlobURL.URL(),
-		metadata,
-		azblob.ModifiedAccessConditions{},
-		azblob.BlobAccessConditions{},
-		azblob.DefaultAccessTier,
-		nil)
+	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	}
+	defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
+
+	dstBlobClient := containerClient.NewBlobClient(dst)
+	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
+		Metadata: metadata,
+	})
+
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
 	}
 
-	copyStatus := copyResp.CopyStatus()
+	copyStatus := *copyResp.CopyStatus
 	for {
-		if copyStatus == azblob.CopyStatusSuccess {
+		if copyStatus == blob.CopyStatusTypeSuccess {
 			if move {
-				_, err = srcBlobURL.Delete(
-					context.Background(),
-					azblob.DeleteSnapshotsOptionNone,
-					azblob.BlobAccessConditions{
-						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
-					})
+				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
+					AccessConditions: &blob.AccessConditions{
+						LeaseAccessConditions: &blob.LeaseAccessConditions{
+							LeaseID: &leaseID,
+						},
+					},
+				})
 				return err
 			}
 			return nil
-		} else if copyStatus == azblob.CopyStatusPending {
+		} else if copyStatus == blob.CopyStatusTypePending {
 			time.Sleep(1 * time.Second)
-			blobPropsResp, err := dstBlobURL.GetProperties(
-				context.Background(),
-				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
-				azblob.ClientProvidedKeyOptions{})
+			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
 			if err != nil {
-				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
+				return fmt.Errorf("error getting copy progress %s", dst)
 			}
-			copyStatus = blobPropsResp.CopyStatus()
+			copyStatus = *getMetadata.CopyStatus
 
-			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
+			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
 			if err != nil {
-				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
+				return fmt.Errorf("error renewing source blob lease %s", src)
 			}
 		} else {
 			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -337,7 +243,9 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 // SymLink creates a copy of src file and adds link information as meta data
 func (storage *PublishedStorage) SymLink(src string, dst string) error {
-	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
+	metadata := make(map[string]*string)
+	metadata["SymLink"] = &src
+	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
 }
 
 // HardLink using symlink functionality as hard links do not exist
@@ -347,29 +255,33 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
-	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	_, err := blobClient.GetProperties(context.Background(), nil)
 	if err != nil {
-		storageError, ok := err.(azblob.StorageError)
-		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+		if isBlobNotFound(err) {
 			return false, nil
 		}
-		return false, err
-	} else if resp.StatusCode() == http.StatusOK {
-		return true, nil
+		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
 	}
-	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+	return true, nil
 }
 
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
-	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	props, err := blobClient.GetProperties(context.Background(), nil)
 	if err != nil {
-		return "", err
-	} else if resp.StatusCode() != http.StatusOK {
-		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+		return "", fmt.Errorf("failed to get blob properties: %v", err)
 	}
-	return resp.NewMetadata()["SymLink"], nil
+
+	metadata := props.Metadata
+	if originalBlob, exists := metadata["original_blob"]; exists {
+		return *originalBlob, nil
+	}
+	return "", fmt.Errorf("error reading link %s: %v", path, err)
 }

azure/public_test.go

@@ -7,8 +7,11 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+        "bytes"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 	. "gopkg.in/check.v1"
@@ -43,14 +46,14 @@ func randString(n int) string {
 func (s *PublishedStorageSuite) SetUpSuite(c *C) {
 	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
 	if s.accountName == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
 	}
 	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 	if s.accountKey == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
@@ -66,8 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-	cnt := s.storage.container
-	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
 	c.Assert(err, IsNil)
 
 	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -75,41 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-	cnt := s.storage.container
-	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
+        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-	blob := s.storage.container.NewBlobURL(path)
-	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
 	c.Assert(err, IsNil)
-	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
-	data, err := ioutil.ReadAll(body)
+	data, err := ioutil.ReadAll(resp.Body)
 	c.Assert(err, IsNil)
 	return data
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-	_, err := s.storage.container.NewBlobURL(path).GetProperties(
-		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+        serviceClient := s.storage.az.client.ServiceClient()
+        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
+        blobClient := containerClient.NewBlobClient(path)
+        _, err := blobClient.GetProperties(context.Background(), nil)
 	c.Assert(err, NotNil)
-	storageError, ok := err.(azblob.StorageError)
+
+        storageError, ok := err.(*azcore.ResponseError)
 	c.Assert(ok, Equals, true)
-	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
+	c.Assert(storageError.StatusCode, Equals, 404)
 }
 
 func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	hash := md5.Sum(data)
-	_, err := azblob.UploadBufferToBlockBlob(
-		context.Background(),
-		data,
-		s.storage.container.NewBlockBlobURL(path),
-		azblob.UploadToBlockBlobOptions{
-			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
-				ContentMD5: hash[:],
-			},
-		})
+        uploadOptions := &azblob.UploadStreamOptions{
+            HTTPHeaders: &blob.HTTPHeaders{
+            	BlobContentMD5: hash[:],
+            },
+        }
+        reader := bytes.NewReader(data)
+        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
 	c.Assert(err, IsNil)
 }
 
@@ -129,7 +132,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
 }
 
 func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
@@ -300,45 +303,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 
 	// first link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
 
 	// for prefixed storage:
 	// first link from pool
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	// 2nd link from pool, providing wrong path for source file
 	//
-	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
+	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with nested file name
-	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))

cmd/api_serve.go

@@ -1,11 +1,15 @@
 package cmd
 
 import (
+	stdcontext "context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/aptly-dev/aptly/api"
 	"github.com/aptly-dev/aptly/systemd/activation"
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}
@@ -55,6 +59,19 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
+	server := http.Server{Handler: api.Router(context)}
+
+	sigchan := make(chan os.Signal, 1)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go (func() {
+		if _, ok := <-sigchan; ok {
+			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
+			context.TaskList().Wait()
+			server.Shutdown(stdcontext.Background())
+		}
+	})()
+	defer close(sigchan)
+
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
@@ -67,19 +84,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		}
 		defer listener.Close()
 
-		err = http.Serve(listener, api.Router(context))
-		if err != nil {
-			return fmt.Errorf("unable to serve: %s", err)
-		}
-		return nil
+		err = server.Serve(listener)
+	} else {
+		server.Addr = listen
+		err = server.ListenAndServe()
 	}
 
-	err = http.ListenAndServe(listen, api.Router(context))
-	if err != nil {
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("unable to serve: %s", err)
 	}
 
-	return err
+	return nil
 }
 
 func makeCmdAPIServe() *commander.Command {

cmd/cmd.go

@@ -118,7 +118,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
 	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {

cmd/config_show.go

@@ -5,19 +5,30 @@ import (
 	"fmt"
 
 	"github.com/smira/commander"
+	"gopkg.in/yaml.v3"
 )
 
-func aptlyConfigShow(cmd *commander.Command, args []string) error {
+func aptlyConfigShow(_ *commander.Command, _ []string) error {
+	showYaml := context.Flags().Lookup("yaml").Value.Get().(bool)
 
 	config := context.Config()
-	prettyJSON, err := json.MarshalIndent(config, "", "    ")
 
-	if err != nil {
-		return fmt.Errorf("unable to dump the config file: %s", err)
+	if showYaml {
+		yamlData, err := yaml.Marshal(&config)
+		if err != nil {
+			return fmt.Errorf("error marshaling to YAML: %s", err)
+		}
+
+		fmt.Println(string(yamlData))
+	} else {
+		prettyJSON, err := json.MarshalIndent(config, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to dump the config file: %s", err)
+		}
+
+		fmt.Println(string(prettyJSON))
 	}
 
-	fmt.Println(string(prettyJSON))
-
 	return nil
 }
 
@@ -35,5 +46,6 @@ Example:
 
 `,
 	}
+	cmd.Flag.Bool("yaml", false, "show yaml config")
 	return cmd
 }

cmd/graph.go

@@ -4,13 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
@@ -36,7 +34,7 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	buf := bytes.NewBufferString(graph.String())
 
-	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	tempfile, err := os.CreateTemp("", "aptly-graph")
 	if err != nil {
 		return err
 	}
@@ -80,10 +78,6 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	defer func() {
-		_ = os.Remove(tempfilename)
-	}()
-
 	if output != "" {
 		err = utils.CopyFile(tempfilename, output)
 		if err != nil {
@@ -91,23 +85,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("Output saved to %s\n", output)
+		_ = os.Remove(tempfilename)
 	} else {
 		command := getOpenCommand()
-		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+		fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename)
 
 		args := strings.Split(command, " ")
 
 		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
 		viewer.Stderr = os.Stderr
-		if err = viewer.Start(); err == nil {
-			// Wait for a second so that the visualizer has a chance to
-			// open the input file. This needs to be done even if we're
-			// waiting for the visualizer as it can be just a wrapper that
-			// spawns a browser tab and returns right away.
-			defer func(t <-chan time.Time) {
-				<-t
-			}(time.After(time.Second))
-		}
+		err = viewer.Start()
 	}
 
 	return err

cmd/mirror.go

@@ -9,18 +9,18 @@ import (
 )
 
 func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
-	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
-		return nil, nil
-	}
-
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -20,6 +20,10 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -42,7 +46,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -59,7 +63,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
@@ -99,7 +103,7 @@ Example:
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")

cmd/mirror_edit.go

@@ -28,10 +28,11 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 	}
 
 	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter = flag.Value.String()
+			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
 		case "with-installer":
@@ -43,6 +44,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		case "archive-url":
 			repo.SetArchiveRoot(flag.Value.String())
 			fetchMirror = true
+		case "ignore-signatures":
+			ignoreSignatures = true
 		}
 	})
 
@@ -69,7 +72,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 		}
 
-		err = repo.Fetch(context.Downloader(), verifier)
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
@@ -101,7 +104,7 @@ Example:
 	}
 
 	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")

cmd/mirror_list.go

@@ -24,7 +24,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return aptlyMirrorListTxt(cmd, args)
 }
 
-func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -65,7 +65,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())

cmd/mirror_show.go

@@ -27,7 +27,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	return aptlyMirrorShowTxt(cmd, args)
 }
 
-func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -93,7 +93,7 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/mirror_update.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 
@@ -41,20 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -161,7 +166,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 					var e error
 
 					// provision download location
-					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+						task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+					} else {
+						var file *os.File
+						file, e = os.CreateTemp("", task.File.Filename)
+						if e == nil {
+							task.TempDownPath = file.Name()
+							file.Close()
+						}
+					}
 					if e != nil {
 						pushError(e)
 						continue
@@ -173,7 +187,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
-						ignoreMismatch)
+						ignoreChecksums)
 					if e != nil {
 						pushError(e)
 						continue
@@ -197,6 +211,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	defer func() {
+		for _, task := range queue {
+			if task.TempDownPath == "" {
+				continue
+			}
+
+			if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+				fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+			}
+		}
+	}()
+
 	// Import downloaded files
 	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 
@@ -241,7 +267,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
 	return err
 }
 

cmd/package_search.go

@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	if len(args) == 1 {
-		q, err = query.Parse(args[0])
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -49,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
 		Long: `
 Command search displays list of packages in whole DB that match package query.
 
+Use '@file' to read query from file or '@-' for stdin.
 If query is not specified, all the packages are displayed.
 
 Example:

cmd/package_show.go

@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -130,6 +134,8 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
     $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'

cmd/publish.go

@@ -34,10 +34,26 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishDrop(),
 			makeCmdPublishList(),
 			makeCmdPublishRepo(),
+			makeCmdPublishShow(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishSource(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
-			makeCmdPublishShow(),
+		},
+	}
+}
+
+func makeCmdPublishSource() *commander.Command {
+	return &commander.Command{
+		UsageLine: "source",
+		Short:     "manage sources of published repository",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSourceAdd(),
+			makeCmdPublishSourceDrop(),
+			makeCmdPublishSourceList(),
+			makeCmdPublishSourceRemove(),
+			makeCmdPublishSourceReplace(),
+			makeCmdPublishSourceUpdate(),
 		},
 	}
 }

cmd/publish_list.go

@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return aptlyPublishListTxt(cmd, args)
 }
 
-func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -34,7 +34,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
 	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
 		if e != nil {
 			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
 				repo.StoragePrefix(), repo.Distribution, repo.Components())
@@ -77,7 +77,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())

cmd/publish_repo.go

@@ -48,8 +48,10 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_show.go

@@ -24,7 +24,7 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	return aptlyPublishShowTxt(cmd, args)
 }
 
-func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]
@@ -52,7 +52,8 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
 
 	fmt.Printf("Sources:\n")
-	for component, sourceID := range repo.Sources {
+	for _, component := range repo.Components() {
+		sourceID := repo.Sources[component]
 		var name string
 		if repo.SourceKind == deb.SourceSnapshot {
 			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
@@ -76,7 +77,7 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]

cmd/publish_snapshot.go

@@ -116,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	origin := context.Flags().Lookup("origin").Value.String()
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -132,6 +133,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 	published.Label = context.Flags().Lookup("label").Value.String()
 	published.Suite = context.Flags().Lookup("suite").Value.String()
+	published.Codename = context.Flags().Lookup("codename").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
 
@@ -148,6 +150,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
 	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
@@ -161,11 +167,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
-			"the same package pool.\n")
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -239,8 +244,10 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_source_add.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if exists {
+			return fmt.Errorf("unable to add: component '%s' has already been added", component)
+		}
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceAdd,
+		UsageLine: "add <distribution> <source>",
+		Short:     "add source components to a published repo",
+		Long: `
+The command adds components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source add -component=contrib wheezy ppa wheezy-contrib
+
+This command assigns the snapshot wheezy-contrib to the component contrib and
+adds it to published repository revision of ppa/wheezy.
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_drop.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published.DropRevision()
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("Source changes have been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishSourceDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceDrop,
+		UsageLine: "drop <distribution>",
+		Short:     "drop pending source component changes of a published repository",
+		Long: `
+Remove all pending changes what would be applied with a subsequent 'aptly publish update'.
+
+Example:
+
+    $ aptly publish source drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_list.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	if published.Revision == nil {
+		return fmt.Errorf("unable to list: no source changes exist")
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishSourceListJSON(published)
+	}
+
+	return aptlyPublishSourceListTxt(published)
+}
+
+func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	fmt.Printf("Sources:\n")
+	for _, component := range revision.Components() {
+		name := revision.Sources[component]
+		fmt.Printf("  %s: %s [%s]\n", component, name, published.SourceKind)
+	}
+
+	return nil
+}
+
+func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	output, err := json.MarshalIndent(revision.SourceList(), "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+func makeCmdPublishSourceList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceList,
+		UsageLine: "list <distribution>",
+		Short:     "lists revision of published repository",
+		Long: `
+Command lists sources of a published repository.
+
+Example:
+
+    $ aptly publish source list wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
+	}
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_remove.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(components) == 0 {
+		return fmt.Errorf("unable to remove: missing components, specify at least one component")
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for _, component := range components {
+		name, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to remove: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		delete(sources, component)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceRemove,
+		UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
+		Short:     "remove source components from a published repo",
+		Long: `
+The command removes source components (snapshot / local repo) from a published repository.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be removed, e.g.:
+
+Example:
+
+	$ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_replace.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+	context.Progress().Printf("Replacing source list...\n")
+	clear(sources)
+
+	for i, component := range components {
+		name := names[i]
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceReplace() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceReplace,
+		UsageLine: "replace <distribution> <source>",
+		Short:     "replace the source components of a published repository",
+		Long: `
+The command replaces the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_update.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceUpdate,
+		UsageLine: "update <distribution> <source>",
+		Short:     "update the source components of a published repository",
+		Long: `
+The command updates the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source update -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -11,7 +11,10 @@ import (
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
-	var err error
+	var (
+		err   error
+		names []string
+	)
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
@@ -23,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	distribution := args[0]
 	param := "."
 
-	var (
-		names    []string
-		snapshot *deb.Snapshot
-	)
-
 	if len(args) == len(components)+2 {
 		param = args[1]
 		names = args[2:]
@@ -42,16 +40,16 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	if published.SourceKind != deb.SourceSnapshot {
-		return fmt.Errorf("unable to update: not a snapshot publish")
+		return fmt.Errorf("unable to switch: not a published snapshot repository")
 	}
 
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	publishedComponents := published.Components()
@@ -63,17 +61,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
 	}
 
+	snapshotCollection := collectionFactory.SnapshotCollection()
 	for i, component := range components {
 		if !utils.StrSliceHasItem(publishedComponents, component) {
-			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+			return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
 		}
 
-		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
+		snapshot, err := snapshotCollection.ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -100,7 +99,11 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -112,14 +115,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
 		if err != nil {
-			return fmt.Errorf("unable to update: %s", err)
+			return fmt.Errorf("unable to switch: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -127,15 +129,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
-		Short:     "update published repository by switching to new snapshot",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
+		Short:     "update published repository by switching to new source",
 		Long: `
-Command switches in-place published snapshots with new snapshot contents. All
+Command switches in-place published snapshots with new source contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
 For multiple component repositories, flag -component should be given with
-list of components to update. Corresponding snapshots should be given in the
+list of components to update. Corresponding sources should be given in the
 same order, e.g.:
 
 	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
@@ -161,6 +163,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -31,18 +31,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != deb.SourceLocalRepo {
-		return fmt.Errorf("unable to update: not a local repository publish")
-	}
-
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	components := published.Components()
-	for _, component := range components {
-		published.UpdateLocalRepo(component)
+	result, err := published.Update(collectionFactory, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -64,7 +60,11 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -76,14 +76,15 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -92,15 +93,21 @@ func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
 		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
-		Short:     "update published local repository",
+		Short:     "update published repository",
 		Long: `
-Command re-publishes (updates) published local repository. <distribution>
-and <prefix> should be occupied with local repository published
-using command aptly publish repo. Update happens in-place with
-minimum possible downtime for published repository.
+The command updates updates a published repository after applying pending changes to the sources.
 
-For multiple component published repositories, all local repositories
-are updated.
+For published local repositories:
+
+    * update to match local repository contents
+
+For published snapshots:
+
+    * switch components to new snapshot
+
+The update happens in-place with minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories are updated.
 
 Example:
 
@@ -119,6 +126,7 @@ Example:
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/repo_add.go

@@ -91,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 func makeCmdRepoAdd() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoAdd,
-		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		UsageLine: "add <name> (<package file.deb>|<directory>)...",
 		Short:     "add packages to local repository",
 		Long: `
 Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.

cmd/repo_include.go

@@ -29,7 +29,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
-	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
 	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
 	collectionFactory := context.NewCollectionFactory()
@@ -83,14 +86,14 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 func makeCmdRepoInclude() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoInclude,
-		UsageLine: "include <file.changes>|<directory> ...",
+		UsageLine: "include (<file.changes>|<directory>)...",
 		Short:     "add packages to local repositories based on .changes files",
 		Long: `
 Command include looks for .changes files in list of arguments or specified directories. Each
 .changes file is verified, parsed, referenced files are put into separate temporary directory
 and added into local repository. Successfully imported files are removed by default.
 
-Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+Additionally uploads could be restricted with 'uploaders.json' file. Rules in this file control
 uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
 
 Example:

cmd/repo_list.go

@@ -24,7 +24,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return aptlyRepoListTxt(cmd, args)
 }
 
-func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -71,7 +71,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())

cmd/repo_move.go

@@ -110,13 +110,24 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 	}
 
-	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+	toProcess, err := srcList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            dstList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -179,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -38,14 +38,18 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
 	}
 
 	list.PrepareIndex()
-	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	toRemove, err := list.Filter(deb.FilterOptions{Queries: queries})
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -81,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/repo_show.go

@@ -25,7 +25,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	return aptlyRepoShowTxt(cmd, args)
 }
 
-func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -58,7 +58,7 @@ func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/serve.go

@@ -29,7 +29,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}

cmd/snapshot_create.go

@@ -84,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotCreate,
-		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		UsageLine: "create <name> (from mirror <mirror-name> | from repo <repo-name> | empty)",
 		Short:     "creates snapshot of mirror (local repository) contents",
 		Long: `
 Command create <name> from mirror makes persistent immutable snapshot of remote

cmd/snapshot_filter.go

@@ -60,14 +60,25 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i, arg := range args[2:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := packageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            nil,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
@@ -96,6 +107,8 @@ Command filter does filtering in snapshot <source>, producing another
 snapshot <destination>. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'

cmd/snapshot_list.go

@@ -23,7 +23,7 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	return aptlySnapshotListTxt(cmd, args)
 }
 
-func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -59,7 +59,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
 	var err error
 
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)

cmd/snapshot_pull.go

@@ -88,7 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-3)
 	for i, arg := range args[3:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -97,7 +101,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := sourcePackageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  !noDeps,
+		Source:            packageList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -112,7 +123,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
 		if !noRemove && !seen {
 			// Remove all packages with the same name and architecture
-			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
 			for _, p := range pS {
 				packageList.Remove(p)
 				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
@@ -160,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server

cmd/snapshot_search.go

@@ -78,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	list.PrepareIndex()
 
 	if len(args) == 2 {
-		q, err = query.Parse(args[1])
+		value, err := GetStringOrFileContent(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -103,8 +107,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 	}
 
-	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := list.Filter(deb.FilterOptions{
+		Queries:           []deb.PackageQuery{q},
+		WithDependencies:  withDeps,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -129,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
 
 If query is not specified, all the packages are displayed.
 
+Use '@file' syntax to read package query from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/snapshot_show.go

@@ -25,7 +25,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	return aptlySnapshotShowTxt(cmd, args)
 }
 
-func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 	var err error
 	name := args[0]
 	collectionFactory := context.NewCollectionFactory()
@@ -85,7 +85,7 @@ func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/string_or_file_flag.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/smira/flag"
+)
+
+// StringOrFileFlag is a custom flag type that can handle both string input and file input.
+// If the input starts with '@', it is treated as a filename and the contents are read from the file.
+// If the input is '@-', the contents are read from stdin.
+type StringOrFileFlag struct {
+	value string
+}
+
+func (s *StringOrFileFlag) String() string {
+	return s.value
+}
+
+func (s *StringOrFileFlag) Set(value string) error {
+	var err error
+	s.value, err = GetStringOrFileContent(value)
+	return err
+}
+
+func (s *StringOrFileFlag) Get() any {
+	return s.value
+}
+
+func AddStringOrFileFlag(flagSet *flag.FlagSet, name string, value string, usage string) *StringOrFileFlag {
+	result := &StringOrFileFlag{value: value}
+	flagSet.Var(result, name, usage)
+	return result
+}
+
+func GetStringOrFileContent(value string) (string, error) {
+	if !strings.HasPrefix(value, "@") {
+		return value, nil
+	}
+
+	filename := strings.TrimPrefix(value, "@")
+	var data []byte
+	var err error
+	if filename == "-" { // Read from stdin
+		data, err = io.ReadAll(os.Stdin)
+	} else {
+		data, err = os.ReadFile(filename)
+	}
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}

cmd/task_run.go

@@ -62,11 +62,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 			text, _ = reader.ReadString('\n')
 			if text == "\n" {
 				break
-			} else {
-				text = strings.TrimSpace(text) + ","
-				parsedArgs, _ := shellwords.Parse(text)
-				cmdArgs = append(cmdArgs, parsedArgs...)
 			}
+			text = strings.TrimSpace(text) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
 		}
 
 		if len(cmdArgs) == 0 {
@@ -132,7 +131,7 @@ func formatCommands(args []string) [][]string {
 func makeCmdTaskRun() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyTaskRun,
-		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		UsageLine: "run (-filename=<filename> | <commands>...)",
 		Short:     "run aptly tasks",
 		Long: `
 Command helps organise multiple aptly commands in one single aptly task, running as single thread.

codecov.yml

@@ -3,5 +3,5 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 2%
         if_ci_failed: error

completion.d/_aptly

@@ -457,6 +457,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-distribution=[distribution name to publish]:distribution:($dists)"
                             "-label=[label to publish]:label: "
                             "-suite=[suite to publish]:suite: "
+                            "-codename=[codename to publish]:codename: "
                             "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
                             "-origin=[origin name to publish]:origin: "
                             ${components_options[@]}

completion.d/aptly

@@ -1,5 +1,3 @@
-#!/bin/bash
-
 # (The MIT License)
 #
 # Copyright (c) 2014 Andrey Smirnov
@@ -56,12 +54,14 @@ _aptly()
 {
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
+    prevprev="${COMP_WORDS[COMP_CWORD-2]}"
 
     commands="api config db graph mirror package publish repo serve snapshot task version"
     options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
     db_subcommands="cleanup recover"
     mirror_subcommands="create drop edit show list rename search update"
-    publish_subcommands="drop list repo snapshot switch update"
+    publish_subcommands="drop list repo snapshot switch update source"
+    publish_source_subcommands="drop list add remove update replace"
     snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
     repo_subcommands="add copy create drop edit import include list move remove rename search show"
     package_subcommands="search show"
@@ -150,6 +150,17 @@ _aptly()
         esac
     fi
 
+    case "$prevprev" in
+        "publish")
+            case "$prev" in
+                "source")
+                COMPREPLY=($(compgen -W "${publish_source_subcommands}" -- ${cur}))
+                return 0
+                ;;
+            esac
+        ;;
+   esac
+
     case "$cmd" in
       "mirror")
         case "$subcmd" in
@@ -503,7 +514,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
@@ -577,6 +588,19 @@ _aptly()
           ;;
         esac
       ;;
+      "source")
+        case "$subcmd" in
+          "add")
+              return 0
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
       "package")
         case "$subcmd" in
           "search")

console/progress.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/cheggaaa/pb"
+	"github.com/rs/zerolog/log"
 	"github.com/wsxiaoys/terminal/color"
 )
 
@@ -34,6 +35,7 @@ type Progress struct {
 	queue    chan printTask
 	bar      *pb.ProgressBar
 	barShown bool
+	worker   ProgressWorker
 }
 
 // Check interface
@@ -42,16 +44,19 @@ var (
 )
 
 // NewProgress creates new progress instance
-func NewProgress() *Progress {
-	return &Progress{
+func NewProgress(structuredLogging bool) *Progress {
+	p := &Progress{
 		stopped: make(chan bool),
 		queue:   make(chan printTask, 100),
 	}
+
+	p.worker = progressWorkerFactroy(structuredLogging, p)
+	return p
 }
 
 // Start makes progress start its work
 func (p *Progress) Start() {
-	go p.worker()
+	go p.worker.run()
 }
 
 // Shutdown shuts down progress display
@@ -69,7 +74,7 @@ func (p *Progress) Flush() {
 }
 
 // InitBar starts progressbar for count bytes or count items
-func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
+func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}
@@ -173,42 +178,94 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 	}
 }
 
-func (p *Progress) worker() {
+type ProgressWorker interface {
+	run()
+}
+
+func progressWorkerFactroy(structuredLogging bool, progress *Progress) ProgressWorker {
+	if structuredLogging {
+		worker := loggerProgressWorker{progress: progress}
+		return &worker
+	}
+
+	worker := standardProgressWorker{progress: progress}
+	return &worker
+}
+
+type standardProgressWorker struct {
+	progress *Progress
+}
+
+func (w *standardProgressWorker) run() {
 	hasBar := false
 
 	for {
-		task := <-p.queue
+		task := <-w.progress.queue
 		switch task.code {
 		case codeBarEnabled:
 			hasBar = true
 		case codeBarDisabled:
 			hasBar = false
 		case codePrint:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 			fmt.Print(task.message)
 		case codePrintStdErr:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 			fmt.Fprint(os.Stderr, task.message)
 		case codeProgress:
 			if hasBar {
 				fmt.Print("\r" + task.message)
-				p.barShown = true
+				w.progress.barShown = true
 			}
 		case codeHideProgress:
-			if p.barShown {
+			if w.progress.barShown {
 				fmt.Print("\r\033[2K")
-				p.barShown = false
+				w.progress.barShown = false
 			}
 		case codeFlush:
 			task.reply <- true
 		case codeStop:
-			p.stopped <- true
+			w.progress.stopped <- true
+			return
+		}
+	}
+}
+
+type loggerProgressWorker struct {
+	progress *Progress
+}
+
+func (w *loggerProgressWorker) run() {
+	hasBar := false
+
+	for {
+		task := <-w.progress.queue
+		switch task.code {
+		case codeBarEnabled:
+			hasBar = true
+		case codeBarDisabled:
+			hasBar = false
+		case codePrint, codePrintStdErr:
+			log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
+		case codeProgress:
+			if hasBar {
+				log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
+				w.progress.barShown = true
+			}
+		case codeHideProgress:
+			if w.progress.barShown {
+				w.progress.barShown = false
+			}
+		case codeFlush:
+			task.reply <- true
+		case codeStop:
+			w.progress.stopped <- true
 			return
 		}
 	}

console/progress_test.go

@@ -0,0 +1,24 @@
+package console
+
+import (
+	"fmt"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ProgressSuite struct {}
+
+var _ = Suite(&ProgressSuite{})
+
+func (s *ProgressSuite) TestNewProgress(c *C) {
+	p := NewProgress(false)
+	c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &standardProgressWorker{}))
+
+	p = NewProgress(true)
+	c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &loggerProgressWorker{}))
+}

context/context.go

@@ -12,12 +12,14 @@ import (
 	"runtime/pprof"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/azure"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/etcddb"
 	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/files"
@@ -48,6 +50,7 @@ type AptlyContext struct {
 	publishedStorages map[string]aptly.PublishedStorage
 	dependencyOptions int
 	architecturesList []string
+	structuredLogging bool
 	// Debug features
 	fileCPUProfile *os.File
 	fileMemProfile *os.File
@@ -93,13 +96,14 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 				Fatal(err)
 			}
 		} else {
-			configLocations := []string{
-				filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-				"/etc/aptly.conf",
-			}
+			homeLocation := filepath.Join(os.Getenv("HOME"), ".aptly.conf")
+			configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
 
 			for _, configLocation := range configLocations {
 				err = utils.LoadConfig(configLocation, &utils.Config)
+				if os.IsPermission(err) || os.IsNotExist(err) {
+					continue
+				}
 				if err == nil {
 					break
 				}
@@ -109,11 +113,13 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 
 			if err != nil {
-				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
-				// as this is fresh aptly installation, we don't need to support legacy pool locations
-				utils.Config.SkipLegacyPool = true
-				utils.SaveConfig(configLocations[0], &utils.Config)
+				utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
+				err = utils.LoadConfig(homeLocation, &utils.Config)
+				if err != nil {
+					Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
+				}
 			}
 		}
 
@@ -195,7 +201,7 @@ func (context *AptlyContext) Progress() aptly.Progress {
 
 func (context *AptlyContext) _progress() aptly.Progress {
 	if context.progress == nil {
-		context.progress = console.NewProgress()
+		context.progress = console.NewProgress(context.structuredLogging)
 		context.progress.Start()
 	}
 
@@ -272,7 +278,7 @@ func (context *AptlyContext) DBPath() string {
 
 // DBPath builds path to database
 func (context *AptlyContext) dbPath() string {
-	return filepath.Join(context.config().RootDir, "db")
+	return filepath.Join(context.config().GetRootDir(), "db")
 }
 
 // Database opens and returns current instance of database
@@ -286,8 +292,18 @@ func (context *AptlyContext) Database() (database.Storage, error) {
 func (context *AptlyContext) _database() (database.Storage, error) {
 	if context.database == nil {
 		var err error
-
-		context.database, err = goleveldb.NewDB(context.dbPath())
+		switch context.config().DatabaseBackend.Type {
+		case "leveldb":
+			dbPath := filepath.Join(context.config().GetRootDir(), "db")
+			if len(context.config().DatabaseBackend.DbPath) != 0 {
+				dbPath = context.config().DatabaseBackend.DbPath
+			}
+			context.database, err = goleveldb.NewDB(dbPath)
+		case "etcd":
+			context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
+		default:
+			context.database, err = goleveldb.NewDB(context.dbPath())
+		}
 		if err != nil {
 			return nil, fmt.Errorf("can't instantiate database: %s", err)
 		}
@@ -344,10 +360,7 @@ func (context *AptlyContext) ReOpenDatabase() error {
 
 // NewCollectionFactory builds factory producing all kinds of collections
 func (context *AptlyContext) NewCollectionFactory() *deb.CollectionFactory {
-	context.Lock()
-	defer context.Unlock()
-
-	db, err := context._database()
+	db, err := context.Database()
 	if err != nil {
 		Fatal(err)
 	}
@@ -360,7 +373,26 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	defer context.Unlock()
 
 	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
+		storageConfig := context.config().PackagePoolStorage
+		if storageConfig.Azure != nil {
+			var err error
+			context.packagePool, err = azure.NewPackagePool(
+				storageConfig.Azure.AccountName,
+				storageConfig.Azure.AccountKey,
+				storageConfig.Azure.Container,
+				storageConfig.Azure.Prefix,
+				storageConfig.Azure.Endpoint)
+			if err != nil {
+				Fatal(err)
+			}
+		} else {
+			poolRoot := context.config().PackagePoolStorage.Local.Path
+			if poolRoot == "" {
+				poolRoot = filepath.Join(context.config().GetRootDir(), "pool")
+			}
+
+			context.packagePool = files.NewPackagePool(poolRoot, !context.config().SkipLegacyPool)
+		}
 	}
 
 	return context.packagePool
@@ -374,7 +406,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 	publishedStorage, ok := context.publishedStorages[name]
 	if !ok {
 		if name == "" {
-			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
+			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().GetRootDir(), "public"), "hardlink", "")
 		} else if strings.HasPrefix(name, "filesystem:") {
 			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
@@ -393,7 +425,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
 				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
 				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
-				params.ForceSigV2, params.Debug)
+				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
 			if err != nil {
 				Fatal(err)
 			}
@@ -432,7 +464,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 // UploadPath builds path to upload storage
 func (context *AptlyContext) UploadPath() string {
-	return filepath.Join(context.Config().RootDir, "upload")
+	return filepath.Join(context.Config().GetRootDir(), "upload")
 }
 
 func (context *AptlyContext) pgpProvider() string {
@@ -456,7 +488,7 @@ func (context *AptlyContext) pgpProvider() string {
 	return provider
 }
 
-func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
+func (context *AptlyContext) getGPGFinder() pgp.GPGFinder {
 	switch context.pgpProvider() {
 	case "gpg1":
 		return pgp.GPG1Finder()
@@ -479,7 +511,7 @@ func (context *AptlyContext) GetSigner() pgp.Signer {
 		return &pgp.GoSigner{}
 	}
 
-	return pgp.NewGpgSigner(context.getGPGFinder(provider))
+	return pgp.NewGpgSigner(context.getGPGFinder())
 }
 
 // GetVerifier returns Verifier with respect to provider
@@ -492,7 +524,12 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
 		return &pgp.GoVerifier{}
 	}
 
-	return pgp.NewGpgVerifier(context.getGPGFinder(provider))
+	return pgp.NewGpgVerifier(context.getGPGFinder())
+}
+
+// SkelPath builds the local skeleton folder
+func (context *AptlyContext) SkelPath() string {
+	return filepath.Join(context.config().GetRootDir(), "skel")
 }
 
 // UpdateFlags sets internal copy of flags in the context
@@ -526,7 +563,7 @@ func (context *AptlyContext) GoContextHandleSignals() {
 
 	// Catch ^C
 	sigch := make(chan os.Signal, 1)
-	signal.Notify(sigch, os.Interrupt)
+	signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
 
 	var cancel gocontext.CancelFunc
 
@@ -540,6 +577,11 @@ func (context *AptlyContext) GoContextHandleSignals() {
 	}()
 }
 
+// StructuredLogging allows to set the structuredLogging flag
+func (context *AptlyContext) StructuredLogging(structuredLogging bool) {
+	context.structuredLogging = structuredLogging
+}
+
 // Shutdown shuts context down
 func (context *AptlyContext) Shutdown() {
 	context.Lock()
@@ -561,6 +603,9 @@ func (context *AptlyContext) Shutdown() {
 			context.fileMemProfile = nil
 		}
 	}
+	if context.taskList != nil {
+		context.taskList.Stop()
+	}
 	if context.database != nil {
 		context.database.Close()
 		context.database = nil

context/context_test.go

@@ -1,6 +1,8 @@
 package context
 
 import (
+	"fmt"
+	"os"
 	"reflect"
 	"testing"
 
@@ -82,5 +84,6 @@ func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
 	// storage never exists.
 	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
 		FatalErrorPanicMatches,
-		&FatalError{ReturnCode: 1, Message: "published local storage fuji not configured"})
+		&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)",
+			os.Getenv("HOME"))})
 }

database/etcddb/batch.go

@@ -0,0 +1,53 @@
+package etcddb
+
+import (
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type EtcDBatch struct {
+	s   *EtcDStorage
+	ops []clientv3.Op
+}
+
+type WriteOptions struct {
+	NoWriteMerge bool
+	Sync         bool
+}
+
+func (b *EtcDBatch) Put(key []byte, value []byte) (err error) {
+	b.ops = append(b.ops, clientv3.OpPut(string(key), string(value)))
+	return
+}
+
+func (b *EtcDBatch) Delete(key []byte) (err error) {
+	b.ops = append(b.ops, clientv3.OpDelete(string(key)))
+	return
+}
+
+func (b *EtcDBatch) Write() (err error) {
+	kv := clientv3.NewKV(b.s.db)
+
+	batchSize := 128
+	for i := 0; i < len(b.ops); i += batchSize {
+		txn := kv.Txn(Ctx)
+		end := i + batchSize
+		if end > len(b.ops) {
+			end = len(b.ops)
+		}
+
+		batch := b.ops[i:end]
+		txn.Then(batch...)
+		_, err = txn.Commit()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	return
+}
+
+// batch should implement database.Batch
+var (
+	_ database.Batch = &EtcDBatch{}
+)

database/etcddb/database.go

@@ -0,0 +1,32 @@
+package etcddb
+
+import (
+	"context"
+	"time"
+
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+var Ctx = context.TODO()
+
+func internalOpen(url string) (cli *clientv3.Client, err error) {
+	cfg := clientv3.Config{
+		Endpoints:            []string{url},
+		DialTimeout:          30 * time.Second,
+		MaxCallSendMsgSize:   2147483647, // (2048 * 1024 * 1024) - 1
+		MaxCallRecvMsgSize:   2147483647,
+		DialKeepAliveTimeout: 7200 * time.Second,
+	}
+
+	cli, err = clientv3.New(cfg)
+	return
+}
+
+func NewDB(url string) (database.Storage, error) {
+	cli, err := internalOpen(url)
+	if err != nil {
+		return nil, err
+	}
+	return &EtcDStorage{url, cli, ""}, nil
+}

database/etcddb/database_test.go

@@ -0,0 +1,158 @@
+package etcddb_test
+
+import (
+	"testing"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/etcddb"
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type EtcDDBSuite struct {
+	url string
+	db  database.Storage
+}
+
+var _ = Suite(&EtcDDBSuite{})
+
+func (s *EtcDDBSuite) SetUpTest(c *C) {
+	var err error
+	s.db, err = etcddb.NewDB("127.0.0.1:2379")
+	c.Assert(err, IsNil)
+}
+
+func (s *EtcDDBSuite) TestSetUpTest(c *C) {
+	var err error
+	s.db, err = etcddb.NewDB("127.0.0.1:2379")
+	c.Assert(err, IsNil)
+}
+
+func (s *EtcDDBSuite) TestGetPut(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+	var err error
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}
+
+func (s *EtcDDBSuite) TestDelete(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	_, err = s.db.Get(key)
+	c.Assert(err, IsNil)
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
+
+}
+
+func (s *EtcDDBSuite) TestByPrefix(c *C) {
+	//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	keys := [][]byte{}
+	values := [][]byte{}
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		keys = append(keys, append([]byte(nil), k...))
+		values = append(values, append([]byte(nil), v...))
+		return nil
+	}), IsNil)
+
+	c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
+
+	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), Equals, database.ErrNotFound)
+
+	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
+		return database.ErrNotFound
+	}), IsNil)
+
+	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+}
+
+func (s *EtcDDBSuite) TestHasPrefix(c *C) {
+	//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
+	//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
+
+	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+
+	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
+	c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
+}
+
+func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
+	var (
+		key    = []byte("key")
+		key2   = []byte("key2")
+		value  = []byte("value")
+		value2 = []byte("value2")
+	)
+	transaction, err := s.db.OpenTransaction()
+
+	err = s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	c.Assert(err, IsNil)
+	transaction.Put(key2, value2)
+	v, err := s.db.Get(key)
+	c.Check(v, DeepEquals, value)
+        err = transaction.Delete(key)
+	c.Assert(err, IsNil)
+
+	_, err = transaction.Get(key2)
+	c.Assert(err, IsNil)
+
+	v2, err := transaction.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = transaction.Get(key)
+	c.Assert(err, IsNil)
+
+	err = transaction.Commit()
+	c.Check(err, IsNil)
+
+	v2, err = transaction.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = transaction.Get(key)
+	c.Assert(err, NotNil)
+}
+

database/etcddb/storage.go

@@ -0,0 +1,202 @@
+package etcddb
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/google/uuid"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type EtcDStorage struct {
+	url       string
+	db        *clientv3.Client
+	tmpPrefix string // prefix for temporary DBs
+}
+
+// CreateTemporary creates new DB of the same type in temp dir
+func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
+	tmp := uuid.NewString()
+	return &EtcDStorage{
+		url:       s.url,
+		db:        s.db,
+		tmpPrefix: tmp,
+	}, nil
+}
+
+func (s *EtcDStorage) applyPrefix(key []byte) []byte {
+	if len(s.tmpPrefix) != 0 {
+		return append([]byte(s.tmpPrefix+"/"), key...)
+	}
+	return key
+}
+
+// Get key value from etcd
+func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
+	realKey := s.applyPrefix(key)
+	getResp, err := s.db.Get(Ctx, string(realKey))
+	if err != nil {
+		return
+	}
+	for _, kv := range getResp.Kvs {
+		value = kv.Value
+		break
+	}
+	if len(value) == 0 {
+		err = database.ErrNotFound
+		return
+	}
+	return
+}
+
+// Put saves key to etcd, if key has the same value in DB already, it is not saved
+func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
+	realKey := s.applyPrefix(key)
+	_, err = s.db.Put(Ctx, string(realKey), string(value))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// Delete removes key from etcd
+func (s *EtcDStorage) Delete(key []byte) (err error) {
+	realKey := s.applyPrefix(key)
+	_, err = s.db.Delete(Ctx, string(realKey))
+	if err != nil {
+		return
+	}
+	return
+}
+
+// KeysByPrefix returns all keys that start with prefix
+func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
+	realPrefix := s.applyPrefix(prefix)
+	result := make([][]byte, 0, 20)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return nil
+	}
+	for _, ev := range getResp.Kvs {
+		key := ev.Key
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, key)
+	}
+	return result
+}
+
+// FetchByPrefix returns all values with keys that start with prefix
+func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
+	realPrefix := s.applyPrefix(prefix)
+	result := make([][]byte, 0, 20)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return nil
+	}
+	for _, kv := range getResp.Kvs {
+		valc := make([]byte, len(kv.Value))
+		copy(valc, kv.Value)
+		result = append(result, kv.Value)
+	}
+
+	return result
+}
+
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
+	realPrefix := s.applyPrefix(prefix)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return false
+	}
+	return getResp.Count > 0
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
+	realPrefix := s.applyPrefix(prefix)
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
+	if err != nil {
+		return err
+	}
+
+	for _, kv := range getResp.Kvs {
+		err := proc(kv.Key, kv.Value)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Close finishes etcd connect
+func (s *EtcDStorage) Close() error {
+	// do not close temporary db
+	if len(s.tmpPrefix) != 0 {
+		return nil
+	}
+	if s.db == nil {
+		return nil
+	}
+	err := s.db.Close()
+	s.db = nil
+	return err
+}
+
+// Reopen tries to open (re-open) the database
+func (s *EtcDStorage) Open() error {
+	if s.db != nil {
+		return nil
+	}
+	var err error
+	s.db, err = internalOpen(s.url)
+	return err
+}
+
+// CreateBatch creates a Batch object
+func (s *EtcDStorage) CreateBatch() database.Batch {
+	if s.db == nil {
+		return nil
+	}
+	return &EtcDBatch{
+		s: s,
+	}
+}
+
+// OpenTransaction creates new transaction.
+func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) {
+	tmpdb, err := s.CreateTemporary()
+	if err != nil {
+		return nil, err
+	}
+	return &transaction{s: s, tmpdb: tmpdb}, nil
+}
+
+// CompactDB does nothing for etcd
+func (s *EtcDStorage) CompactDB() error {
+	return nil
+}
+
+// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys)
+func (s *EtcDStorage) Drop() error {
+	if len(s.tmpPrefix) != 0 {
+		getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix())
+		if err != nil {
+			return nil
+		}
+		for _, kv := range getResp.Kvs {
+			_, err = s.db.Delete(Ctx, string(kv.Key))
+			if err != nil {
+				return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key)
+			}
+		}
+	}
+	return nil
+}
+
+// Check interface
+var (
+	_ database.Storage = &EtcDStorage{}
+)

database/etcddb/transaction.go

@@ -0,0 +1,75 @@
+package etcddb
+
+import (
+	"github.com/aptly-dev/aptly/database"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+type transaction struct {
+	s     *EtcDStorage
+	tmpdb database.Storage
+	ops   []clientv3.Op
+}
+
+// Get implements database.Reader interface.
+func (t *transaction) Get(key []byte) (value []byte, err error) {
+	value, err = t.tmpdb.Get(key)
+	// if not found, search main db
+	if err != nil {
+		value, err = t.s.Get(key)
+	}
+	return
+}
+
+// Put implements database.Writer interface.
+func (t *transaction) Put(key, value []byte) (err error) {
+	err = t.tmpdb.Put(key, value)
+	if err != nil {
+		return
+	}
+	t.ops = append(t.ops, clientv3.OpPut(string(key), string(value)))
+	return
+}
+
+// Delete implements database.Writer interface.
+func (t *transaction) Delete(key []byte) (err error) {
+	err = t.tmpdb.Delete(key)
+	if err != nil {
+		return
+	}
+	t.ops = append(t.ops, clientv3.OpDelete(string(key)))
+	return
+}
+
+func (t *transaction) Commit() (err error) {
+	kv := clientv3.NewKV(t.s.db)
+
+	batchSize := 128
+	for i := 0; i < len(t.ops); i += batchSize {
+		txn := kv.Txn(Ctx)
+		end := i + batchSize
+		if end > len(t.ops) {
+			end = len(t.ops)
+		}
+
+		batch := t.ops[i:end]
+		txn.Then(batch...)
+		_, err = txn.Commit()
+		if err != nil {
+			panic(err)
+		}
+	}
+	t.ops = []clientv3.Op{}
+
+	return
+}
+
+// Discard is safe to call after Commit(), it would be no-op
+func (t *transaction) Discard() {
+	t.ops = []clientv3.Op{}
+	t.tmpdb.Drop()
+	return
+}
+
+// transaction should implement database.Transaction
+var _ database.Transaction = &transaction{}

database/goleveldb/storage.go

@@ -3,7 +3,6 @@ package goleveldb
 import (
 	"bytes"
 	"errors"
-	"io/ioutil"
 	"os"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -19,7 +18,7 @@ type storage struct {
 
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *storage) CreateTemporary() (database.Storage, error) {
-	tempdir, err := ioutil.TempDir("", "aptly")
+	tempdir, err := os.MkdirTemp("", "aptly")
 	if err != nil {
 		return nil, err
 	}