debian: fix build

This reverts commit e2cbd637b8.

# Conflicts:
#	azure/public.go
#	go.sum

# Conflicts:
#	go.mod
#	go.sum

.dockerignore

@@ -0,0 +1,10 @@
+.go/
+.git/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+unit.out
+aptly.test
+build/
+dpkgs/

.flake8

@@ -1,7 +1,7 @@
 [flake8]
-max-line-length = 200
+max-line-length = 240
 ignore = E126,E241,E741,W504
 include =
     system
 exclude =
-    system/env
+    system/env

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,6 +4,10 @@ Fixes #
 
 All new code should be covered with tests, documentation should be updated. CI should pass.
 
+Also, to speed up things, if you could kindly "Allow edits and access to secrets by maintainers" in the
+PR settings, as this allows us to rebase the PR on master, fix conflicts, run coverage and help with
+implementing code and tests.
+
 ## Description of the Change
 
 <!--
@@ -14,6 +18,7 @@ Why this change is important?
 
 ## Checklist
 
+- [ ] allow Maintainers to edit PR (rebase, run coverage, help with tests, ...)
 - [ ] unit-test added (if change is algorithm)
 - [ ] functional test added/updated (if change is functional)
 - [ ] man page updated (if applicable)

.github/workflows/ci.yml

@@ -1,5 +1,3 @@
-# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
-
 name: CI
 
 on:
@@ -13,32 +11,41 @@ on:
 defaults:
   run:
     # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
-    shell: bash --noprofile --norc -eo pipefail -x {0}
+    shell: bash --noprofile --norc -eo pipefail {0}
 
 env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
-  build:
-    name: test
-    runs-on: ubuntu-20.04
-    continue-on-error: ${{ matrix.allow_failure }}
+  unit-test:
+    name: "Unit Tests"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+      - name: "Docker Image"
+        run: |
+          make docker-image
+      - name: "Unit Tests"
+        run: |
+          make docker-unit-tests
+          mkdir -p out/coverage
+          mv unit.out out/coverage/
+      - uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests-coverage
+          path: out/
+
+  test:
+    name: "System Test"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
     timeout-minutes: 30
-    strategy:
-      fail-fast: true
-      matrix:
-        go: [1.16, 1.17, 1.18]
-        run_long_tests: [no]
-        allow_failure: [false]
-        include:
-          # Disable this for now as it's not clear how to select the latest master branch
-          # version of go using actions/setup-go@v2.
-          # - go: master
-          #   run_long_tests: no
-          #   allow_failure: true
-          - go: 1.17
-            run_long_tests: yes
-            allow_failure: false
 
     env:
       NO_FTP_ACCESS: yes
@@ -47,101 +54,322 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      - name: Setup Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go }}
-
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: v1.45.0
-
-      - name: Setup Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Install O/S packages
+      - name: "Install Test Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
 
-      - name: Install Python packages
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Run flake8"
         run: |
-          pip install six packaging appdirs virtualenv
-          pip install -U pip setuptools
-          pip install -r system/requirements.txt
+          make flake8
 
-      - name: Install Azurite
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Install Azurite"
         id: azuright
         uses: potatoqualitee/azuright@v1.1
         with:
           directory: ${{ runner.temp }}
 
-      - name: Get aptly version
+      - name: "Run Benchmark"
         run: |
-           make version
+          mkdir -p out/coverage
+          COVERAGE_DIR=$PWD/out/coverage make bench
 
-      - name: Make
+      - name: "Run System Tests"
         env:
-          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
-          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
           AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          make
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          mkdir -p out/coverage
+          COVERAGE_DIR=$PWD/out/coverage make system-test
 
-      - name: Upload code coverage
-        if: matrix.run_long_tests
-        uses: codecov/codecov-action@v2
+      - uses: actions/upload-artifact@v4
+        with:
+          name: system-tests-coverage
+          path: out/
+
+  coverage:
+    name: "Upload Coverage"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    needs:
+      - unit-test
+      - test
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Download Unit Test Coverage"
+        uses: actions/download-artifact@v4
+        with:
+          name: unit-tests-coverage
+
+      - name: "Download System Test Coverage"
+        uses: actions/download-artifact@v4
+        with:
+          name: system-tests-coverage
+
+      - name: "Merge Code Coverage"
+        run: |
+          # go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
+          # ~/go/bin/gocovmerge coverage/*.out > coverage.txt
+          awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
+
+      - name: "Upload Code Coverage"
+        if: github.actor != 'dependabot[bot]'
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.txt
+          fail_ci_if_error: true
 
-  release:
-    name: release
-    needs: build
-    runs-on: ubuntu-20.04
+
+  ci-debian-build:
+    name: "Build"
+    needs:
+      - coverage
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        arch: ["amd64", "i386" , "arm64" , "armhf"]
+        include:
+          - name: "Debian 13/trixie"
+            suite: trixie
+            image: debian:trixie-slim
+          - name: "Debian 12/bookworm"
+            suite: bookworm
+            image: debian:bookworm-slim
+          - name: "Debian 11/bullseye"
+            suite: bullseye
+            image: debian:bullseye-slim
+          - name: "Ubuntu 24.04"
+            suite: noble
+            image: ubuntu:24.04
+          - name: "Ubuntu 22.04"
+            suite: jammy
+            image: ubuntu:22.04
+          - name: "Ubuntu 20.04"
+            suite: focal
+            image: ubuntu:20.04
+    container:
+      image: ${{ matrix.image }}
+      options: --user root
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - name: "Install Build Packages"
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
+                          binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf \
+                          libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
+                          gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
         with:
-          # fetch the whole repot for `git describe` to
-          # work and get the nightly verion
+          # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: Setup Go
-        uses: actions/setup-go@v2
+      - name: "Read Go Version"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
 
-      - name: Make Release
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Build Debian packages"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make dpkg DEBARCH=${{ matrix.arch }}
+
+      - name: "Check aptly credentials"
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          found=no
+          if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
+            found=yes
+          fi
+          echo "Aptly credentials available: $found"
+          echo "FOUND=$found" >> $GITHUB_OUTPUT
+        id: aptlycreds
+
+      - name: "Publish CI release to aptly"
+        if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
+
+      - name: "Publish release to aptly"
+        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Upload CI Artifacts"
+        if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags')
+        uses: actions/upload-artifact@v4
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }}
+          path: build/
+          retention-days: 7
+
+  ci-binary-build:
+    name: "Build"
+    needs:
+      - coverage
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, freebsd, darwin]
+        goarch: ["386", "amd64", "arm", "arm64"]
+        exclude:
+          - goos: darwin
+            goarch: 386
+          - goos: darwin
+            goarch: arm
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read Go Version"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
         env:
           GOBIN: /usr/local/bin
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
         run: |
-          make release
+          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
 
-      - name: Publish nightly release to aptly
-        if: github.ref == 'refs/heads/master'
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh nightly
-
-      - name: Publish release to aptly
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
         if: startsWith(github.event.ref, 'refs/tags')
-        env:
-          APTLY_USER: ${{ secrets.APTLY_USER }}
-          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
-        run: |
-          ./upload-artifacts.sh release
-
-      - name: Upload artifacts to GitHub Release
-        if: startsWith(github.event.ref, 'refs/tags')
-        uses: softprops/action-gh-release@v1
         with:
-          body: Release ${{ github.ref }} generated by the CI.
-          files: build/*
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+
+      - name: "Upload CI Artifacts"
+        uses: actions/upload-artifact@v4
+        if: "!startsWith(github.event.ref, 'refs/tags')"
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+          retention-days: 7
+
+  gh-release:
+    name: "Github Release"
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: ci-binary-build
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Download Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: out/
+
+      - name: "Create Release Notes"
+        run: |
+          echo -e "## Changes\n\n" > out/release-notes.md
+          dpkg-parsechangelog -S Changes | tail -n +4 >> out/release-notes.md
+
+      - name: "Release"
+        uses: softprops/action-gh-release@v2
+        with:
+          name: "Aptly Release ${{ steps.releaseversion.outputs.VERSION }}"
+          files: "out/**/aptly_*.zip"
+          body_path: "out/release-notes.md"

.github/workflows/golangci-lint.yml

@@ -0,0 +1,72 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+
+      - name: Install and initialize swagger
+        run: |
+          go install github.com/swaggo/swag/cmd/swag@latest
+          swag init -q --propertyStrategy pascalcase --markdownFiles docs
+        shell: sh
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.64.5
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/scripts/upload-artifacts.sh

@@ -0,0 +1,170 @@
+#!/bin/sh
+
+set -e
+
+builds=build/
+packages=${builds}*.deb
+folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
+aptly_user="$APTLY_USER"
+aptly_password="$APTLY_PASSWORD"
+aptly_api="https://aptly-ops.aptly.info"
+version=`make version`
+
+action=$1
+dist=$2
+
+usage() {
+    echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
+    echo "       $0 release" >&2
+}
+
+# repos and publish must be created beforehand:
+#!/bin/sh
+#for dist in buster bullseye bookworm focal jammy noble
+#do
+#        for build in ci release
+#        do
+#            echo
+#            echo "# Creating and publishing $build/$dist"
+#            aptly repo create -distribution=$dist -component=main aptly-$build-$dist
+#            aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
+#                               -distribution=$dist -batch -keyring=aptly.pub \
+#                               aptly-$build-$dist \
+#                               s3:repo.aptly.info:$build
+#       done
+#done
+
+if [ -z "$action" ]; then
+    usage
+    exit 1
+fi
+
+if [ "action" = "ci" ] && [ -z "$dist" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
+    usage
+    echo Error: please set APTLY_USER and APTLY_PASSWORD
+    exit 1
+fi
+
+echo "Publishing version '$version' to $action for $dist...\n"
+
+upload()
+{
+    echo "\nUploading files:"
+    for file in $packages; do
+        echo " - $file"
+        jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+    done
+}
+
+cleanup() {
+    echo "\nCleanup..."
+    jsonret=`curl -fsS -X DELETE  -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+}
+trap cleanup EXIT
+
+sleeptime=5
+retries=60
+wait_task()
+{
+    _id=$1
+    _success=0
+    sleep $sleeptime
+    for t in `seq $retries`
+    do
+        jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
+        _state=`echo $jsonret | jq .State`
+        if [ "$_state" = "2" ]; then
+            _success=1
+            curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id
+            break
+        fi
+        if [ "$_state" = "3" ]; then
+            echo Error: task failed
+            return 1
+        fi
+        sleep $sleeptime
+    done
+    if [ "$_success" -ne 1 ]; then
+        echo Error: task timeout
+        return 1
+    fi
+    return 0
+}
+
+add_packages() {
+    _aptly_repository=$1
+    _folder=$2
+    jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$_aptly_repository/file/$_folder?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: adding packages to $_aptly_repository failed"
+        exit 1
+    fi
+}
+
+update_publish() {
+    _publish=$1
+    _dist=$2
+    jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+        '{"AcquireByHash": true, "MultiDist": true,
+          "Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
+        -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: publish failed"
+        exit 1
+    fi
+}
+
+if [ "$action" = "ci" ]; then
+    if echo "$version" | grep -vq "+"; then
+       # skip ci when on release tag
+       exit 0
+    fi
+
+    aptly_repository=aptly-ci-$dist
+    aptly_published=s3:repo.aptly.info:ci
+
+elif [ "$action" = "release" ]; then
+    aptly_repository=aptly-release-$dist
+    aptly_published=s3:repo.aptly.info:release
+fi
+
+upload
+
+echo "\nAdding packages to $aptly_repository ..."
+add_packages $aptly_repository $folder
+
+echo "\nUpdating published repo $aptly_published ..."
+update_publish $aptly_published $dist
+
+# if [ "$action" = "OBSOLETErelease" ]; then
+#     aptly_repository=aptly-release
+#     aptly_snapshot=aptly-$version
+#     aptly_published=s3:repo.aptly.info:./squeeze
+#
+#     echo "\nAdding packages to $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
+#     echo
+#
+#     echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
+#         "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
+#     echo
+#
+#     echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
+#     curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+#         "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
+#                                    \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
+#                                                  \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
+#         -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
+#     echo
+# fi
+

.gitignore

@@ -2,11 +2,14 @@
 *.o
 *.a
 *.so
+unit.out
 
 # Folders
 _obj
 _test
 
+tmp/
+
 # Architecture specific extensions/prefixes
 *.[568vq]
 [568vq].out
@@ -23,6 +26,8 @@ _testmain.go
 *.test
 
 coverage.txt
+coverage.out
+coverage.html
 
 *.pyc
 
@@ -32,13 +37,42 @@ root/
 man/aptly.1.html
 man/aptly.1.ronn
 
-.goxc.local.json
-
 system/env/
 
 # created by make build for release artifacts
+VERSION
+aptly.test
+
 build/
 
-pgp/keyrings/aptly2*.gpg
-pgp/keyrings/aptly2*.gpg~
-pgp/keyrings/.#*
+system/files/aptly2.gpg~
+system/files/aptly2_passphrase.gpg~
+
+*.creds
+
+.go/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+
+# debian
+debian/.debhelper/
+debian/aptly.substvars
+debian/aptly/
+debian/debhelper-build-stamp
+debian/files
+debian/aptly-api/
+debian/*.debhelper
+debian/*.debhelper.log
+debian/aptly-api.substvars
+debian/aptly-dbg.substvars
+debian/aptly-dbg/
+usr/bin/aptly
+dpkgs/
+debian/changelog.dpkg-bak
+
+docs/docs.go
+docs/swagger.json
+docs/swagger.yaml
+docs/swagger.conf

.golangci.yml

@@ -1,19 +1,11 @@
-run:
-  tests: false
-
-
+version: "2"
 linters:
-  disable-all: true
-  enable:
-   - deadcode
-   - goconst
-   - gofmt
-   - goimports
-   - govet
-   - ineffassign
-   - misspell
-   - revive
-   - staticcheck
-   - structcheck
-   - varcheck
-   - vetshadow
+  settings:
+    staticcheck:
+      checks:
+      - "all"
+      - "-QF1004"  # could use strings.ReplaceAll instead
+      - "-QF1012"  # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))
+      - "-QF1003"  # could use tagged switch
+      - "-ST1000"  # at least one file in a package should have a package comment
+      - "-QF1001"  # could apply De Morgan's law

.goxc.json

@@ -1,45 +0,0 @@
-{
-	"AppName": "aptly",
-	"ArtifactsDest": "xc-out/",
-	"TasksExclude": [
-		"rmbin",
-		"go-test",
-		"go-vet"
-	],
-    "TasksAppend": [
-		"bintray"
-    ],
-    "TaskSettings": {
-        "debs": {
-            "metadata": {
-                "maintainer": "Andrey Smirnov",
-                "maintainer-email": "me@smira.ru",
-                "description": "Debian repository management tool"
-            },
-            "metadata-deb": {
-                "License": "MIT",
-                "Homepage": "https://www.aptly.info/",
-                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-				"Suggests": "graphviz"
-            },
-            "other-mapped-files": {
-                "/": "root/"
-            }
-        },
-        "bintray": {
-            "repository": "aptly",
-            "subject": "smira",
-            "package": "aptly",
-            "downloadspage": "bintray.md"
-        }
-    },
-	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
-	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
-	"MainDirsExclude": "_man,vendor",
-	"BuildSettings": {
-		"LdFlagsXVars": {
-			"Version": "main.Version"
-		}
-	},
-	"ConfigVersion": "0.9"
-}

.travis.yml

@@ -1,106 +0,0 @@
-dist: xenial
-sudo: required
-
-language: go
-
-go_import_path: github.com/aptly-dev/aptly
-
-addons:
-  apt:
-    packages:
-    - python-virtualenv
-    - graphviz
-    - gnupg2
-    - gpgv2
-
-env:
-  global:
-    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
-    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
-    - NO_FTP_ACCESS: "yes"
-    - BOTO_CONFIG: /dev/null
-    - GO111MODULE: "on"
-    - GOPROXY: https://proxy.golang.org
-
-matrix:
-  allow_failures:
-    - go: master
-      env: RUN_LONG_TESTS=no
-  fast_finish: true
-  include:
-    - go: 1.11.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.12.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.13.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.14.x
-      env: RUN_LONG_TESTS=yes
-    - go: 1.15.x
-      env:
-        - RUN_LONG_TESTS=yes
-        - DEPLOY_BINARIES=yes
-        - APTLY_USER=aptly
-        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
-    - go: master
-      env: RUN_LONG_TESTS=no
-
-before_install:
-  - virtualenv system/env
-  - . system/env/bin/activate
-  - pip install six packaging appdirs
-  - pip install -U pip setuptools
-  - pip install -r system/requirements.txt
-  - make version
-
-install:
-  - make prepare
-
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
-
-notifications:
-  webhooks:
-    urls:
-      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false
-
-before_deploy:
-  - make release
-
-deploy:
-  - provider: releases
-    api_key:
-      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
-    file_glob: true
-    file: build/*
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: s3
-    access_key_id:
-      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
-    secret_access_key:
-      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
-    bucket: aptly-nightly
-    skip_cleanup: true
-    acl: public-read
-    local_dir: build
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh nightly
-    skip_cleanup: true
-    on:
-      branch: master
-      condition: "$DEPLOY_BINARIES = yes"
-  - provider: script
-    script: bash upload-artifacts.sh release
-    skip_cleanup: true
-    on:
-      tags: true
-      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -49,3 +49,38 @@ List of contributors, in chronological order:
 * Samuel Mutel (https://github.com/smutel)
 * Russell Greene (https://github.com/russelltg)
 * Wade Simmons (https://github.com/wadey)
+* Steven Stone (https://github.com/smstone)
+* Josh Bayfield (https://github.com/jbayfield)
+* Boxjan (https://github.com/boxjan)
+* Mauro Regli (https://github.com/reglim)
+* Alexander Zubarev (https://github.com/strike)
+* Nicolas Dostert (https://github.com/acdn-ndostert)
+* Ryan Gonzalez (https://github.com/refi64)
+* Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramon N.Rodriguez (https://github.com/runitonmetal)
+* Golf Hu (https://github.com/hudeng-go)
+* Cookie Fei (https://github.com/wuhuang26)
+* Andrey Loukhnov (https://github.com/aol-nnov)
+* Christoph Fiehe (https://github.com/cfiehe)
+* Blake Kostner (https://github.com/btkostner)
+* Leigh London (https://github.com/leighlondon)
+* Gordian Schoenherr (https://github.com/schoenherrg)
+* Silke Hofstra (https://github.com/silkeh)
+* Itay Porezky (https://github.com/itayporezky)
+* Alejandro Guijarro Monerris (https://github.com/alguimodd)
+* JupiterRider (https://github.com/JupiterRider)
+* Agustin Henze (https://github.com/agustinhenze)
+* Tobias Assarsson (https://github.com/daedaluz)
+* Yaksh Bariya (https://github.com/thunder-coding)
+* Juan Calderon-Perez (https://github.com/gaby)
+* Ato Araki (https://github.com/atotto)
+* Roman Lebedev (https://github.com/LebedevRI)
+* Brian Witt (https://github.com/bwitt)
+* Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)
+* Zhang Xiao (https://github.com/xzhang1)
+* Tom Nguyen (https://github.com/lecafard)
+* Philip Cramer (https://github.com/PhilipCramer)

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 :+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
 
-The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
 These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
 ## What should I know before I get started?
@@ -11,7 +11,7 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
 
 This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
 By participating, you are expected to uphold this code.
-Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
 
 ### List of Repositories
 
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your First Code Contribution
+### Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,63 +68,117 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
 Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
 if that is possible.
 
+### Get the Source
+
+To clone the git repo, run the following commands:
+```
+git clone git@github.com:aptly-dev/aptly.git
+cd aptly
+```
+
 ## Development Setup
 
-This section describes local setup to start contributing to aptly source.
+Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
 
-### Go & Python
+### Docker Development Setup
 
-You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+This section describes the docker setup to start contributing to aptly.
 
-If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
-initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+#### Dependencies
 
-Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
-to prepend it or to skip this test if you're security conscious.
+Install the following on your development machine:
+- docker
+- make
+- git
 
-### Forking and Cloning
+##### Docker installation on macOS
+1. Install [Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install/) (or via [Homebrew](https://brew.sh/))
+2. Allow directory sharing
+   - Open Docker Desktop
+   - Go to `Settings → Resources → File Sharing → Virtual File Shares`
+   - Add the aptly git repository path to the shared list (eg. /home/Users/john/aptly)
 
-As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
+#### Create docker container
 
-    git clone git@github.com:aptly-dev/aptly.git
-    cd aptly
+To build the development docker image, run:
+```
+make docker-image
+```
 
-For main repo under your GitHub user and add it as another Git remote:
+#### Build aptly
 
-    git remote add <user> git@github.com:<user>/aptly.git
+To build the aptly in the development docker container, run:
+```
+make docker-build
+```
 
-That way you can continue to build project as is (you don't need to adjust import paths), but you would need
-to specify your remote name when pushing branches:
+#### Running aptly commands
 
-    git push <user> <your-branch>
+To run aptly commands in the development docker container, run:
+```
+make docker-shell
+```
 
-### Dependencies
+Example:
+```
+$ make docker-shell
+aptly@b43e8473ef81:/work/src$ aptly version
+aptly version: 1.5.0+189+g0fc90dff
+```
 
-You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+#### Running unit tests
 
-    make prepare dev system/env
+In order to run aptly unit tests, enter the following:
+```
+make docker-unit-tests
+```
 
-This is usually one-time action.
+#### Running system tests
 
-Aptly is using Go modules to manage dependencies, download modules using:
+In order to run aptly system tests, enter the following:
+```
+make docker-system-tests
+```
 
-    make modules
+#### Running golangci-lint
 
-### Building
+In order to run aptly unit tests, run:
+```
+make docker-lint
+```
 
-If you want to build aptly binary from your current source tree, run:
+#### More info
+
+Run `make help` for more information.
+
+
+### Local Development Setup
+
+This section describes local setup to start contributing to aptly.
+
+#### Dependencies
+
+Building aptly requires go version 1.22.
+
+On Debian bookworm with backports enabled, go can be installed with:
+
+    apt install -t bookworm-backports golang-go
+
+#### Building
+
+To build aptly, run:
+
+    make build
+
+Run aptly:
+
+    build/aptly
+
+To install aptly into `$GOPATH/bin`, run:
 
     make install
 
-This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
-
-    aptly
-
-Or, if it's not on your path:
-
-    ~/go/bin/aptly
-
-### Unit-tests
+#### Unit-tests
 
 aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
 feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -133,7 +187,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
 
     make test
 
-### Functional Tests
+#### Functional Tests
 
 Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
 runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -180,26 +234,6 @@ There are some packages available under `system/files/` directory which are used
 this default location. You can run aptly under different user or by using non-default config location with non-default
 aptly root directory.
 
-### Style Checks
-
-Style checks could be run with:
-
-    make check
-
-aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
-for the linter could be found in [.golangci.yml](.golangci.yml) file.
-
-Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
-
-### Vendored Code
-
-aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
-repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
-packages over packages in `$GOPATH`.
-
-If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
-Usually all you need is `dep ensure` or `dep ensure -update`.
-
 ### man Page
 
 aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
@@ -217,25 +251,3 @@ Bash and Zsh completion for aptly reside in the same repo under in [completion.d
 When new option or command is introduced, bash completion should be updated to reflect that change.
 
 When aptly package is being built, it automatically pulls bash completion and man page into the package.
-
-## Design
-
-This section requires future work.
-
-*TBD*
-
-### Database
-
-### Package Pool
-
-### Package
-
-### PackageList, PackageRefList
-
-### LocalRepo, RemoteRepo, Snapshot
-
-### PublishedRepository
-
-### Context
-
-### Collections, CollectionFactory

Makefile

@@ -1,81 +1,244 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
-ifdef TRAVIS_TAG
-	TAG=$(TRAVIS_TAG)
-else
-	TAG="$(shell git describe --tags --always)"
-endif
-VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files gpg http query swift s3 utils
+GOPATH=$(shell go env GOPATH)
+VERSION=$(shell make -s version)
 PYTHON?=python3
-TESTS?=
 BINPATH?=$(GOPATH)/bin
-RUN_LONG_TESTS?=yes
+GOLANGCI_LINT_VERSION=v2.0.2  # version supporting go 1.24
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-all: modules test bench check system-test
+export PODMAN_USERNS = keep-id
+DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
 
-prepare:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+# Setting TZ for certificates
+export TZ=UTC
+# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
+export TEST_FAKETIME := 2025-01-02 03:04:05
 
-modules:
-	go mod download
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
+else
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
+endif
+
+# export CAPUTRE=1 for regenrating test gold files
+ifeq ($(CAPTURE),1)
+CAPTURE_ARG := --capture
+endif
+
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+prepare:  ## Install go module dependencies
+	# Prepare go modules
 	go mod verify
 	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-dev:
-	go get -u github.com/laher/goxc
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
+	@reltype=ci ; \
+	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
+	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
+		gittag=`git describe --tags --exact-match 2>/dev/null` ;\
+		if echo "$$gittag" | grep -q '^v[0-9]'; then \
+			reltype=release ; \
+		fi ; \
+	fi ; \
+	echo $$reltype
 
-check: system/env
-ifeq ($(RUN_LONG_TESTS), yes)
-	golangci-lint run
-	system/env/bin/flake8
-endif
+version:  ## Print aptly version
+	@ci="" ; \
+	if [ "`make -s releasetype`" = "ci" ]; then \
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
+	else \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
+	fi
+
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
+	# Generate swagger.conf
+	cp docs/swagger.conf.tpl docs/swagger.conf
+	echo "// @version $(VERSION)" >> docs/swagger.conf
+
+azurite-start:
+	azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
+	echo $$! > ~/.azurite.pid
+
+azurite-stop:
+	@kill `cat ~/.azurite.pid`
+
+swagger: swagger-install
+	# Generate swagger docs
+	@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+
+etcd-install:
+	# Install etcd
+	test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint: prepare
+	# Install golangci-lint
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
 
 install:
-	go install -v -ldflags "-X main.Version=$(VERSION)"
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	# go generate
+	@go generate
+	# go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system/env: system/requirements.txt
-ifeq ($(RUN_LONG_TESTS), yes)
-	rm -rf system/env
-	$(PYTHON) -m venv system/env
-	system/env/bin/pip install -r system/requirements.txt
-endif
+test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify which tests to run)
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
+	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
+	faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/aptly-etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
 
-system-test: install system/env
-ifeq ($(RUN_LONG_TESTS), yes)
+system-test: prepare swagger etcd-install  ## Run system tests
+	# build coverage binary
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
-endif
-
-test:
-	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
+	# Run system tests
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
 
 bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
 	go test -v ./deb -run=nothing -bench=. -benchmem
 
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
+
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
+	fi
+	# clean
+	rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
+	# Run dpkg-buildpackage
+	@buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
+	echo "$$cmd" ; \
+	$$cmd
+	lintian ../*_$(DEBARCH).changes || true
+	# cleanup
+	@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-image-no-cache:  ## Build aptly-dev docker image (no cache)
+	@docker build --no-cache -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-tests:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
+	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		test TEST=$(TEST) \
+		azurite-stop
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
+		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
+		azurite-stop
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
+
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc: dev
-	rm -rf root/
-	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
-	cp man/aptly.1 root/usr/share/man/man1
-	cp completion.d/aptly root/etc/bash_completion.d/
-	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
-	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
-
-release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
-release: goxc
-	rm -rf build/
-	mkdir -p build/
-	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
-
-man:
+man:  ## Create man pages
 	make -C man
 
-version:
-	@echo $(VERSION)
+clean:  ## remove local build and module cache
+	# Clean all generated and build files
+	test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
+	rm -rf .go/
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: man modules version release goxc
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -1,7 +1,3 @@
-=====
-aptly
-=====
-
 .. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
     :target: https://github.com/aptly-dev/aptly/actions
 
@@ -9,20 +5,23 @@ aptly
     :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly
 
+aptly
+=====
+
 Aptly is a swiss army knife for Debian repository management.
 
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
-mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
+open `issues <https://github.com/aptly-dev/aptly/issues>`_ or `discussions <https://github.com/aptly-dev/aptly/discussions>`_.
 
-Aptly features: ("+" means planned features)
+Aptly features:
 
 * make mirrors of remote Debian/Ubuntu repositories, limiting by components/architectures
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
@@ -32,53 +31,70 @@ Aptly features: ("+" means planned features)
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
 * REST API for remote access
-* mirror repositories "as-is" (without resigning with user's key) (+)
-* support for yum repositories (+)
 
-Current limitations:
+Any contributions are welcome! Please see `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
-* translations are not supported yet
+Installation
+=============
 
-Download
---------
+Aptly can be installed on several operating systems.
 
-To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
+Debian / Ubuntu
+----------------
 
-    deb http://repo.aptly.info/ squeeze main
+Aptly is provided in the following debian packages:
 
-And import key that is used to sign the release::
+* **aptly**: Includes the main Aptly binary, man pages, and shell completions
+* **aptly-api**: A systemd service for the REST API, using the global /etc/aptly.conf
+* **aptly-dbg**: Debug symbols for troubleshooting
 
-    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
+The packages can be installed on official `Debian <https://packages.debian.org/search?keywords=aptly>`_ and `Ubuntu <https://packages.ubuntu.com/search?keywords=aptly>`_ distributions.
 
-After that you can install aptly as any other software package::
+Upstream Debian Packages
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    $ apt-get update
-    $ apt-get install aptly
+If a newer version (not available in Debian/Ubuntu) of aptly is required, upstream debian packages (built from git tags) can be installed as follows:
 
-Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
-Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+Install the following APT key (as root)::
 
-If you would like to use nightly builds (unstable), please use following repository::
+    wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
 
-    deb http://repo.aptly.info/ nightly main
+Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
 
-Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
 
-If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
+Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
 
-    git clone https://github.com/aptly-dev/aptly
-    cd aptly
-    make modules install
+Install aptly packages::
 
-Binary would be installed to ``$GOPATH/bin/aptly``.
+    apt-get update
+    apt-get install aptly
+    apt-get install aptly-api  # REST API systemd service
 
-Contributing
-------------
+CI Builds
+~~~~~~~~~~
 
-Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+For testing new features or bugfixes, recent builds are available as CI builds (built from master, may be unstable!) and can be installed as follows:
+
+Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
+
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
+
+Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
+
+Note: same gpg key is used as for the Upstream Debian Packages.
+
+Other Operating Systems
+------------------------
+
+Binary executables (depends almost only on libc) are available on `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_ for:
+
+- macOS / darwin (amd64, arm64)
+- FreeBSD (amd64, arm64, 386, arm)
+- Generic Linux (amd64, arm64, 386, arm)
 
 Integrations
-------------
+=============
 
 Vagrant:
 
@@ -95,10 +111,8 @@ With configuration management systems:
 
 -   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
     (Heavy Water Operations, LLC)
--   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
-    Government Digital Services
--   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
-    TubeMogul
+-   `Puppet module <https://github.com/voxpupuli/puppet-aptly>`_ by
+    Vox Pupuli
 -   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
     Forrest Alvarez and Brian Jackson
 -   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
@@ -115,3 +129,7 @@ GUI for aptly API:
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar
+
+Molior:
+
+-   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth

Releasing.md

@@ -0,0 +1,18 @@
+# Creating a Release
+
+- create branch release/1.x.y
+- update debian/changelog
+- create PR, merge when approved
+- on updated master, create release:
+```
+version=$(dpkg-parsechangelog -S Version)
+echo Releasing prod version $version
+git tag -a v$version -m 'aptly: release $version'
+git push origin v$version master
+```
+- run swagger locally (`make docker-serve`)
+- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
+- add new version to select tag in content/doc/api/swagger.md line 48
+- update version in content/download.md
+- push commit to master
+- create release announcement on https://github.com/aptly-dev/aptly/discussions

api/api.go

@@ -3,17 +3,18 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
 // Lock order acquisition (canonical):
@@ -22,9 +23,69 @@ import (
 //  3. SnapshotCollection
 //  4. PublishedRepoCollection
 
-// GET /api/version
+type aptlyVersion struct {
+	// Aptly Version
+	Version string `json:"Version"`
+}
+
+// @Summary Aptly Version
+// @Description **Get aptly version**
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/version
+// @Description {"Version":"0.9~dev"}
+// @Description ```
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyVersion
+// @Router /api/version [get]
 func apiVersion(c *gin.Context) {
-	c.JSON(200, gin.H{"Version": aptly.Version})
+	version := aptlyVersion{
+		Version: aptly.Version,
+	}
+	c.JSON(200, version)
+}
+
+type aptlyStatus struct {
+	// Aptly Status
+	Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"`
+}
+
+// @Summary Get Ready State
+// @Description **Get aptly ready state**
+// @Description
+// @Description Return aptly ready state:
+// @Description - `Aptly is ready` (HTTP 200)
+// @Description - `Aptly is unavailable` (HTTP 503)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus "Aptly is ready"
+// @Failure 503 {object} aptlyStatus "Aptly is unavailable"
+// @Router /api/ready [get]
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		status := aptlyStatus{Status: "Aptly is ready"}
+		c.JSON(200, status)
+	}
+}
+
+// @Summary Get Health State
+// @Description **Get aptly health state**
+// @Description
+// @Description Return aptly health state:
+// @Description - `Aptly is healthy` (HTTP 200)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus
+// @Router /api/healthy [get]
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
 }
 
 type dbRequestKind int
@@ -108,7 +169,7 @@ func runTaskInBackground(name string, resources []string, proc task.Process) (ta
 			return nil, err
 		}
 
-		defer releaseDatabaseConnection()
+		defer func() { _ = releaseDatabaseConnection() }()
 		return proc(out, detail)
 	})
 }
@@ -117,18 +178,18 @@ func truthy(value interface{}) bool {
 	if value == nil {
 		return false
 	}
-	switch value.(type) {
+	switch v := value.(type) {
 	case string:
-		switch strings.ToLower(value.(string)) {
+		switch strings.ToLower(v) {
 		case "n", "no", "f", "false", "0", "off":
 			return false
 		default:
 			return true
 		}
 	case int:
-		return !(value.(int) == 0)
+		return v != 0
 	case bool:
-		return value.(bool)
+		return v
 	}
 	return true
 }
@@ -137,20 +198,29 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 	// Run this task in background if configured globally or per-request
 	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
 	if background {
-		log.Println("Executing task asynchronously")
+		log.Debug().Msg("Executing task asynchronously")
 		task, conflictErr := runTaskInBackground(name, resources, proc)
 		if conflictErr != nil {
-			c.AbortWithError(409, conflictErr)
+			AbortWithJSONError(c, 409, conflictErr)
 			return
 		}
 		c.JSON(202, task)
 	} else {
-		log.Println("Executing task synchronously")
-		out := context.Progress()
-		detail := task.Detail{}
-		retValue, err := proc(out, &detail)
+		log.Debug().Msg("Executing task synchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		_, _ = context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
+		_, _ = context.TaskList().DeleteTaskByID(task.ID)
 		if err != nil {
-			c.AbortWithError(retValue.Code, err)
+			AbortWithJSONError(c, retValue.Code, err)
 			return
 		}
 		if retValue != nil {
@@ -168,7 +238,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -176,7 +246,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -193,23 +263,59 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			Source:            nil,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		_ = list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter(deb.FilterOptions{
+					Queries: []deb.PackageQuery{versionQ},
+				})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						_ = tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						_ = list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -219,3 +325,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 		c.JSON(200, list.Strings())
 	}
 }
+
+func AbortWithJSONError(c *gin.Context, code int, err error) {
+	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+	_ = c.AbortWithError(code, err)
+}

api/api_test.go

@@ -1,16 +1,20 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
-	ctx "github.com/aptly-dev/aptly/context"
-	"github.com/gin-gonic/gin"
+	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"strings"
 	"testing"
 
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
 	"github.com/smira/flag"
 
 	. "gopkg.in/check.v1"
@@ -20,33 +24,37 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ApiSuite struct {
+type APISuite struct {
 	context    *ctx.AptlyContext
 	flags      *flag.FlagSet
 	configFile *os.File
 	router     http.Handler
 }
 
-var _ = Suite(&ApiSuite{})
+var _ = Suite(&APISuite{})
 
 func createTestConfig() *os.File {
-	file, err := ioutil.TempFile("", "aptly")
+	file, err := os.CreateTemp("", "aptly")
 	if err != nil {
 		return nil
 	}
 	jsonString, err := json.Marshal(gin.H{
-		"architectures": []string{},
+		"architectures":         []string{},
+		"enableMetricsEndpoint": true,
 	})
 	if err != nil {
 		return nil
 	}
-	file.Write(jsonString)
+	_, _ = file.Write(jsonString)
 	return file
 }
 
-func (s *ApiSuite) SetUpSuite(c *C) {
+func (s *APISuite) setupContext() error {
+	aptly.Version = "testVersion"
 	file := createTestConfig()
-	c.Assert(file, NotNil)
+	if nil == file {
+		return fmt.Errorf("unable to create the test configuration file")
+	}
 	s.configFile = file
 
 	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
@@ -57,24 +65,33 @@ func (s *ApiSuite) SetUpSuite(c *C) {
 	s.flags = flags
 
 	context, err := ctx.NewContext(s.flags)
-	c.Assert(err, IsNil)
+	if nil != err {
+		return err
+	}
 
 	s.context = context
 	s.router = Router(context)
+
+	return nil
 }
 
-func (s *ApiSuite) TearDownSuite(c *C) {
-	os.Remove(s.configFile.Name())
+func (s *APISuite) SetUpSuite(c *C) {
+	err := s.setupContext()
+	c.Assert(err, IsNil)
+}
+
+func (s *APISuite) TearDownSuite(c *C) {
+	_ = os.Remove(s.configFile.Name())
 	s.context.Shutdown()
 }
 
-func (s *ApiSuite) SetUpTest(c *C) {
+func (s *APISuite) SetUpTest(c *C) {
 }
 
-func (s *ApiSuite) TearDownTest(c *C) {
+func (s *APISuite) TearDownTest(c *C) {
 }
 
-func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
 	w := httptest.NewRecorder()
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
@@ -85,14 +102,55 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
 	return w, nil
 }
 
-func (s *ApiSuite) TestGetVersion(c *C) {
+func (s *APISuite) TestGinRunsInReleaseMode(c *C) {
+	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
+}
+
+func (s *APISuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, ".*Version.*")
+	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
 }
 
-func (s *ApiSuite) TestTruthy(c *C) {
+func (s *APISuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *APISuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *APISuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
+}
+
+func (s *APISuite) TestRepoCreate(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name": "dummy",
+	})
+	c.Assert(err, IsNil)
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+}
+
+func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy("no"), Equals, false)
 	c.Check(truthy("n"), Equals, false)
 	c.Check(truthy("off"), Equals, false)

api/db.go

@@ -11,9 +11,17 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// POST /api/db/cleanup
-func apiDbCleanup(c *gin.Context) {
-
+// @Summary DB Cleanup
+// @Description **Cleanup Aptly DB**
+// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore.
+// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos.
+// @Tags Database
+// @Produce json
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/db/cleanup [post]
+func apiDBCleanup(c *gin.Context) {
 	resources := []string{string(task.AllResourcesKey)}
 	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 		var err error
@@ -101,8 +109,8 @@ func apiDbCleanup(c *gin.Context) {
 
 		if toDelete.Len() > 0 {
 			batch := db.CreateBatch()
-			toDelete.ForEach(func(ref []byte) error {
-				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+			_ = toDelete.ForEach(func(ref []byte) error {
+				_ = collectionFactory.PackageCollection().DeleteByKey(ref, batch)
 				return nil
 			})
 

api/error.go

@@ -0,0 +1,5 @@
+package api
+
+type Error struct {
+	Error string `json:"error"`
+}

api/files.go

@@ -6,10 +6,17 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 func verifyPath(path string) bool {
 	path = filepath.Clean(path)
 	for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -24,27 +31,37 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.AbortWithError(400, fmt.Errorf("wrong dir"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
 	return true
 }
 
-// GET /files
+// @Summary List Directories
+// @Description **Get list of upload directories**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description  $ curl http://localhost:8080/api/files
+// @Description  ["aptly-0.9"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Success 200 {array} string "List of files"
+// @Router /api/files [get]
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -53,76 +70,146 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, list)
 }
 
-// POST /files/:dir/
+// @Summary Upload Files
+// @Description **Upload files to a directory**
+// @Description
+// @Description - one or more files can be uploaded
+// @Description - existing uploaded are overwritten
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Accept multipart/form-data
+// @Param dir path string true "Directory to upload files to. Created if does not exist"
+// @Param files formData file true "Files to upload"
+// @Produce json
+// @Success 200 {array} string "list of uploaded files"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [post]
 func apiFilesUpload(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	stored := []string{}
+	openFiles := []*os.File{}
 
+	// Write all files first
 	for _, files := range c.Request.MultipartForm.File {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.AbortWithError(500, err)
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
+				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer src.Close()
 
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.AbortWithError(500, err)
+				_ = src.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
+				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
+			_ = src.Close()
 			if err != nil {
-				c.AbortWithError(500, err)
+				_ = dst.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
+			// Keep file open for batch sync
+			openFiles = append(openFiles, dst)
 			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
 		}
 	}
 
-	c.JSON(200, stored)
+	// Sync all files at once to catch ENOSPC errors
+	for i, dst := range openFiles {
+		err := syncFile(dst)
+		if err != nil {
+			// Close all files
+			for _, f := range openFiles {
+				_ = f.Close()
+			}
+			AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
+			return
+		}
+	}
 
+	// Close all files
+	for _, dst := range openFiles {
+		_ = dst.Close()
+	}
+
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
+	c.JSON(200, stored)
 }
 
-// GET /files/:dir
+// @Summary List Files
+// @Description **Show uploaded files in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to list"
+// @Success 200 {array} string "Files found in directory"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [get]
 func apiFilesListFiles(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
 	list := []string{}
-	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	listLock := &sync.Mutex{}
+	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -131,6 +218,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil
@@ -138,9 +227,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 		} else {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 		}
 		return
 	}
@@ -148,36 +237,66 @@ func apiFilesListFiles(c *gin.Context) {
 	c.JSON(200, list)
 }
 
-// DELETE /files/:dir
+// @Summary Delete Directory
+// @Description **Delete upload directory and uploaded files within**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [delete]
 func apiFilesDeleteDir(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, gin.H{})
 }
 
-// DELETE /files/:dir/:name
+// @Summary Delete File
+// @Description **Delete a uploaded file in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to delete from"
+// @Param name path string true "File to delete"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir}/{name} [delete]
 func apiFilesDeleteFile(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	if !verifyPath(c.Params.ByName("name")) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+	dir := utils.SanitizePath(c.Params.ByName("dir"))
+	name := utils.SanitizePath(c.Params.ByName("name"))
+	if !verifyPath(name) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 	}

api/files_test.go

@@ -0,0 +1,476 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+type FilesUploadDiskFullSuite struct {
+	aptlyContext *ctx.AptlyContext
+	flags        *flag.FlagSet
+	configFile   *os.File
+	router       http.Handler
+}
+
+var _ = Suite(&FilesUploadDiskFullSuite{})
+
+func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
+	aptly.Version = "testVersion"
+
+	file, err := os.CreateTemp("", "aptly")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+		"rootDir":       c.MkDir(),
+	})
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+	_ = file.Close()
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	aptlyContext, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.aptlyContext = aptlyContext
+	s.router = Router(aptlyContext)
+	context = aptlyContext
+}
+
+func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.aptlyContext != nil {
+		s.aptlyContext.Shutdown()
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
+	testContent := []byte("test file content for upload")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "testfile.txt")
+	c.Assert(err, IsNil)
+
+	_, err = part.Write(testContent)
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
+	testContent := bytes.Repeat([]byte("A"), 10000)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "largefile.bin")
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(part, bytes.NewReader(testContent))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir2", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(len(content), Equals, len(testContent))
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
+	testFiles := map[string][]byte{
+		"file1.txt": []byte("content of file 1"),
+		"file2.txt": bytes.Repeat([]byte("B"), 5000),
+		"file3.deb": []byte("debian package content"),
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	for filename, content := range testFiles {
+		part, err := writer.CreateFormFile("file", filename)
+		c.Assert(err, IsNil)
+		_, err = part.Write(content)
+		c.Assert(err, IsNil)
+	}
+
+	err := writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/multitest", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
+	for filename, expectedContent := range testFiles {
+		uploadedFile := filepath.Join(uploadDir, filename)
+		content, err := os.ReadFile(uploadedFile)
+		c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
+		c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
+	oldSyncFile := syncFile
+	syncFile = func(f *os.File) error {
+		if filepath.Base(f.Name()) == "syncfail.txt" {
+			return syscall.ENOSPC
+		}
+		return nil
+	}
+	defer func() { syncFile = oldSyncFile }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part1, err := writer.CreateFormFile("file", "ok.txt")
+	c.Assert(err, IsNil)
+	_, err = part1.Write([]byte("ok"))
+	c.Assert(err, IsNil)
+
+	part2, err := writer.CreateFormFile("file", "syncfail.txt")
+	c.Assert(err, IsNil)
+	_, err = part2.Write([]byte("will fail on sync"))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+	c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
+	c.Check(verifyPath("a/b/c"), Equals, true)
+	c.Check(verifyPath("../x"), Equals, false)
+	c.Check(verifyPath("./x"), Equals, true)
+	c.Check(verifyPath(".."), Equals, false)
+	c.Check(verifyPath("."), Equals, false)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
+	uploadRoot := s.aptlyContext.UploadPath()
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "d1"), Equals, true)
+	c.Check(strings.Contains(body, "d2"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
+	req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 404)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files/dir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "a.txt"), Equals, true)
+	c.Check(strings.Contains(body, "b.txt"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(base)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(filepath.Join(base, "a.txt"))
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
+	req, err := http.NewRequest("DELETE", "/api/files/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.Chmod(base, 0), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	req, err := http.NewRequest("GET", "/api/files/noperms", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	subdir := filepath.Join(base, "subdir")
+	c.Assert(os.MkdirAll(subdir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
+	req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/..", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
+	// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
+	form := &multipart.Form{
+		File: map[string][]*multipart.FileHeader{
+			"file": {{Filename: "broken.bin"}},
+		},
+	}
+
+	req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
+	c.Assert(err, IsNil)
+	req.MultipartForm = form
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.Chmod(base, 0555), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/readonly", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
+	parent := s.aptlyContext.UploadPath()
+	base := filepath.Join(parent, "cantremove")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	c.Assert(os.Chmod(parent, 0555), IsNil)
+	defer func() { _ = os.Chmod(parent, 0777) }()
+
+	req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}

api/gpg.go

@@ -1,32 +1,84 @@
 package api
 
 import (
+	"bufio"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
 
-// POST /api/gpg
-func apiGPGAddKey(c *gin.Context) {
-	var b struct {
-		Keyserver   string
-		GpgKeyID    string
-		GpgKeyArmor string
-		Keyring     string
-	}
+type gpgKeyInfo struct {
+	// 16-character key ID (short form)
+	KeyID string `json:"KeyID"         example:"8B48AD6246925553"`
+	// Full fingerprint
+	Fingerprint string `json:"Fingerprint"   example:"D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0"`
+	// Key validity (u=unknown, f=fulltrust, m=marginal, n=never)
+	Validity string `json:"Validity"      example:"u"`
+	// User ID(s) associated with this key
+	UserIDs []string `json:"UserIDs"       example:"John Doe <john@example.com>"`
+	// Creation date (Unix timestamp format from gpg)
+	CreatedAt string `json:"CreatedAt"     example:"2023-01-15"`
+}
 
+type gpgKeyListResponse struct {
+	Keys []gpgKeyInfo `json:"Keys"`
+}
+
+type gpgAddKeyParams struct {
+	// Keyring for adding the keys (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Add ASCII armored gpg public key, do not download from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+
+	// Keyserver to download keys provided in `GpgKeyID`
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// Keys do download from `Keyserver`, separated by space
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500 8B48AD6246925553"`
+}
+
+type gpgDeleteKeyParams struct {
+	// Keyring to delete keys from (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Key ID or fingerprint to delete
+	GpgKeyID string `json:"GpgKeyID"        example:"8B48AD6246925553"`
+}
+
+// @Summary Add GPG Keys
+// @Description **Adds GPG keys to aptly keyring**
+// @Description
+// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description
+// @Description Keys can be added in two ways:
+// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
+// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
+// @Description
+// @Tags Mirrors
+// @Consume  json
+// @Param request body gpgAddKeyParams true "Parameters"
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/gpg/key [post]
+func apiGPGAddKey(c *gin.Context) {
+	b := gpgAddKeyParams{}
 	if c.Bind(&b) != nil {
 		return
 	}
+	b.Keyserver = utils.SanitizePath(b.Keyserver)
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
+	// b.Keyring can be an absolute path
 
 	var err error
-	args := []string{"--no-default-keyring"}
+	args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
 	keyring := "trustedkeys.gpg"
 	if len(b.Keyring) > 0 {
 		keyring = b.Keyring
@@ -37,21 +89,21 @@ func apiGPGAddKey(c *gin.Context) {
 	}
 	if len(b.GpgKeyArmor) > 0 {
 		var tempdir string
-		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
-		defer os.RemoveAll(tempdir)
+		defer func() { _ = os.RemoveAll(tempdir) }()
 
 		keypath := filepath.Join(tempdir, "key")
 		keyfile, e := os.Create(keypath)
 		if e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 			return
 		}
 		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
-			c.AbortWithError(400, e)
+			AbortWithJSONError(c, 400, e)
 		}
 		args = append(args, "--import", keypath)
 
@@ -62,10 +114,10 @@ func apiGPGAddKey(c *gin.Context) {
 		args = append(args, keys...)
 	}
 
-	finder := pgp.GPG1Finder()
+	finder := pgp.GPGDefaultFinder()
 	gpg, _, err := finder.FindGPG()
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -74,11 +126,186 @@ func apiGPGAddKey(c *gin.Context) {
 	// there is no error handling for such as gpg will do this for us
 	cmd := exec.Command(gpg, args...)
 	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
-	cmd.Stdout = os.Stdout
-	if err = cmd.Run(); err != nil {
-		c.AbortWithError(400, err)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		c.JSON(400, string(out))
 		return
 	}
 
-	c.JSON(200, gin.H{})
+	c.JSON(200, string(out))
+}
+
+// @Summary List GPG Keys
+// @Description **Lists all GPG keys in aptly keyring**
+// @Description
+// @Description Returns all public keys currently installed in the aptly GPG keyring.
+// @Description
+// @Tags Mirrors
+// @Param keyring query string false "Keyring file to list keys from (default: trustedkeys.gpg)" example(trustedkeys.gpg)
+// @Produce json
+// @Success 200 {object} gpgKeyListResponse "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/gpg/keys [get]
+func apiGPGListKeys(c *gin.Context) {
+	keyring := c.DefaultQuery("keyring", "trustedkeys.gpg")
+	keyring = utils.SanitizePath(keyring)
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	args := []string{
+		"--no-default-keyring",
+		"--with-colons",
+		"--keyring", keyring,
+		"--list-keys",
+	}
+
+	cmd := exec.Command(gpg, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("failed to list keys: %s", string(out)))
+		return
+	}
+
+	keys := parseGPGOutput(string(out))
+	c.JSON(200, gpgKeyListResponse{Keys: keys})
+}
+
+// @Summary Delete GPG Key
+// @Description **Deletes a GPG key from aptly keyring**
+// @Description
+// @Description Removes a public key from the aptly GPG keyring. This is useful for removing
+// @Description compromised keys or cleaning up obsolete keys.
+// @Description
+// @Tags Mirrors
+// @Consume  json
+// @Param request body gpgDeleteKeyParams true "Parameters"
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/gpg/key [delete]
+func apiGPGDeleteKey(c *gin.Context) {
+	b := gpgDeleteKeyParams{}
+	if c.Bind(&b) != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("invalid request body"))
+		return
+	}
+
+	if len(strings.TrimSpace(b.GpgKeyID)) == 0 {
+		AbortWithJSONError(c, 400, fmt.Errorf("GpgKeyID is required"))
+		return
+	}
+
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	// b.Keyring can be an absolute path
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	args := []string{
+		"--no-default-keyring",
+		"--allow-non-selfsigned-uid",
+		"--batch",
+		"--yes",
+	}
+
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+
+	args = append(args, "--keyring", keyring)
+	args = append(args, "--delete-keys", b.GpgKeyID)
+
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("failed to delete key: %s", string(out)))
+		return
+	}
+
+	c.JSON(200, string(out))
+}
+
+// parseGPGOutput parses the output of `gpg --with-colons --list-keys`
+// and returns a structured list of keys
+func parseGPGOutput(output string) []gpgKeyInfo {
+	var keys []gpgKeyInfo
+	var currentKey *gpgKeyInfo
+
+	scanner := bufio.NewScanner(strings.NewReader(output))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+
+		parts := strings.Split(line, ":")
+		if len(parts) < 10 {
+			continue
+		}
+
+		recordType := parts[0]
+
+		// pub: public key record
+		if recordType == "pub" {
+			// Save previous key if it exists
+			if currentKey != nil && currentKey.KeyID != "" {
+				keys = append(keys, *currentKey)
+			}
+
+			// Create new key entry
+			// Format: pub:trust:length:algo:keyid:created:expires:uidhash:...
+			keyID := parts[4]
+			if len(keyID) >= 16 {
+				keyID = keyID[len(keyID)-16:] // Last 16 chars = short key ID
+			}
+			validity := parts[1]
+			createdAt := parts[5]
+
+			currentKey = &gpgKeyInfo{
+				KeyID:      keyID,
+				Validity:   validity,
+				CreatedAt:  createdAt,
+				UserIDs:    []string{},
+				Fingerprint: "",
+			}
+		}
+
+		// uid: user ID record
+		if recordType == "uid" && currentKey != nil {
+			// Format: uid:trust:created:expires:keyid:uidhash:uidtype:validity:userID:...
+			if len(parts) >= 10 {
+				userID := parts[9]
+				if userID != "" {
+					currentKey.UserIDs = append(currentKey.UserIDs, userID)
+				}
+			}
+		}
+
+		// fpr: fingerprint record
+		if recordType == "fpr" && currentKey != nil {
+			// Format: fpr:::::::::fingerprint:
+			if len(parts) >= 10 {
+				fingerprint := parts[9]
+				currentKey.Fingerprint = fingerprint
+			}
+		}
+	}
+
+	// Don't forget the last key
+	if currentKey != nil && currentKey.KeyID != "" {
+		keys = append(keys, *currentKey)
+	}
+
+	return keys
 }

api/gpg_test.go

@@ -0,0 +1,451 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	. "gopkg.in/check.v1"
+)
+
+type GPGSuite struct {
+	APISuite
+}
+
+var _ = Suite(&GPGSuite{})
+
+func (s *GPGSuite) withFakeGPG(c *C, scriptBody string, test func(scriptPath string)) {
+	tempDir, err := os.MkdirTemp("", "aptly-fake-gpg")
+	c.Assert(err, IsNil)
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	scriptPath := filepath.Join(tempDir, "gpg")
+	err = os.WriteFile(scriptPath, []byte(scriptBody), 0o755)
+	c.Assert(err, IsNil)
+
+	oldPath := os.Getenv("PATH")
+	err = os.Setenv("PATH", tempDir+string(os.PathListSeparator)+oldPath)
+	c.Assert(err, IsNil)
+	defer func() { _ = os.Setenv("PATH", oldPath) }()
+
+	test(scriptPath)
+}
+
+func (s *GPGSuite) fakeGPGScript(c *C, listOutput string, deleteOutput string, deleteError string) string {
+	return "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then\n" +
+		"  echo 'gpg (GnuPG) 2.2.27'\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"args=\"$*\"\n" +
+		"if printf '%s' \"$args\" | grep -q -- '--list-keys'; then\n" +
+		"  cat <<'EOF'\n" + listOutput + "\nEOF\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"if printf '%s' \"$args\" | grep -q -- '--delete-keys'; then\n" +
+		"  if [ -n \"" + strings.ReplaceAll(deleteError, "\n", "") + "\" ]; then\n" +
+		"    echo '" + strings.ReplaceAll(deleteError, "'", "'\\''") + "'\n" +
+		"    exit 1\n" +
+		"  fi\n" +
+		"  cat <<'EOF'\n" + deleteOutput + "\nEOF\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"echo 'unexpected invocation' >&2\n" +
+		"exit 1\n"
+}
+
+// TestParseGPGOutputEmpty tests parsing of empty GPG output
+func (s *GPGSuite) TestParseGPGOutputEmpty(c *C) {
+	output := ""
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 0)
+}
+
+// TestParseGPGOutputSingleKeyMinimal tests parsing a single key with minimal fields
+func (s *GPGSuite) TestParseGPGOutputSingleKeyMinimal(c *C) {
+	// Minimal valid GPG output with one key
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.KeyID, Equals, "8B48AD6246925553")
+	c.Check(key.Validity, Equals, "u")
+	c.Check(key.CreatedAt, Equals, "1611864000")
+	c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+	c.Check(key.UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
+}
+
+// TestParseGPGOutputMultipleKeys tests parsing multiple keys
+func (s *GPGSuite) TestParseGPGOutputMultipleKeys(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+pub:f:2048:1:A1B2C3D4E5F67890:1580592000:1612128000:uidhash:::scESC:::::::23::0:
+uid:f::::1580592000::0987654321::Jane Smith <jane@example.com>::::::::::0:
+fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 2)
+
+	// First key
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+	c.Check(keys[0].Validity, Equals, "u")
+	c.Check(keys[0].UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
+
+	// Second key
+	c.Check(keys[1].KeyID, Equals, "A1B2C3D4E5F67890")
+	c.Check(keys[1].Validity, Equals, "f")
+	c.Check(keys[1].UserIDs, DeepEquals, []string{"Jane Smith <jane@example.com>"})
+}
+
+// TestParseGPGOutputMultipleUIDs tests a key with multiple user IDs
+func (s *GPGSuite) TestParseGPGOutputMultipleUIDs(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+uid:u::::1611864000::1234567891::John Doe <john.doe@company.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.UserIDs, HasLen, 2)
+	c.Check(key.UserIDs, DeepEquals, []string{
+		"John Doe <john@example.com>",
+		"John Doe <john.doe@company.com>",
+	})
+}
+
+// TestParseGPGOutputMalformedLines tests that malformed lines are skipped
+func (s *GPGSuite) TestParseGPGOutputMalformedLines(c *C) {
+	// Mix of valid and invalid lines (too few fields)
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+invalid:line:with:only:three:fields
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+}
+
+// TestParseGPGOutputEmptyLines tests that empty lines are skipped
+func (s *GPGSuite) TestParseGPGOutputEmptyLines(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+}
+
+// TestParseGPGOutputKeyWithoutUID tests a public key without user ID
+func (s *GPGSuite) TestParseGPGOutputKeyWithoutUID(c *C) {
+	// Key without uid record (should still be included)
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.KeyID, Equals, "8B48AD6246925553")
+	c.Check(key.UserIDs, HasLen, 0)
+	c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+}
+
+// TestParseGPGOutputVariousValidity tests different validity values
+func (s *GPGSuite) TestParseGPGOutputVariousValidity(c *C) {
+	output := `pub:u:4096:1:KEY1111111111111:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::Key1::::::::::0:
+fpr:::::::::1111111111111111111111111111111111111111:
+pub:f:4096:1:KEY2222222222222:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:f::::1611864000::1234567891::Key2::::::::::0:
+fpr:::::::::2222222222222222222222222222222222222222:
+pub:m:4096:1:KEY3333333333333:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:m::::1611864000::1234567892::Key3::::::::::0:
+fpr:::::::::3333333333333333333333333333333333333333:
+pub:n:4096:1:KEY4444444444444:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:n::::1611864000::1234567893::Key4::::::::::0:
+fpr:::::::::4444444444444444444444444444444444444444:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 4)
+
+	validities := []string{"u", "f", "m", "n"}
+	for i, validity := range validities {
+		c.Check(keys[i].Validity, Equals, validity)
+	}
+}
+
+// TestParseGPGOutputShortKeyID tests that key IDs are shortened to 16 chars
+func (s *GPGSuite) TestParseGPGOutputShortKeyID(c *C) {
+	// 40-character key ID that should be shortened to last 16 chars
+	longKeyID := "0123456789ABCDEF0123456789ABCDEF8B48AD62"
+	output := `pub:u:4096:1:` + longKeyID + `:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should extract the last 16 characters: 89ABCDEF8B48AD62
+	c.Check(keys[0].KeyID, Equals, "89ABCDEF8B48AD62")
+}
+
+// TestParseGPGOutputSpecialCharactersInUID tests user IDs with special characters
+func (s *GPGSuite) TestParseGPGOutputSpecialCharactersInUID(c *C) {
+	// UID with Unicode characters and special formatting
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::J\xc3\xb6hn D\xc3\xb6\xc3\xa9 (D\xc3\xbcss) <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should preserve the encoded special characters
+	c.Check(keys[0].UserIDs, HasLen, 1)
+}
+
+// TestAPIGPGListKeysDefaultKeyring tests the HTTP endpoint with default keyring
+func (s *GPGSuite) TestAPIGPGListKeysDefaultKeyring(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, `pub:u:4096:1:8B48AD6246925553:1611864000:::::
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`, "", ""), func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		var result gpgKeyListResponse
+		err = json.NewDecoder(response.Body).Decode(&result)
+		c.Assert(err, IsNil)
+		c.Check(result.Keys, HasLen, 1)
+		c.Check(result.Keys[0].KeyID, Equals, "8B48AD6246925553")
+	})
+}
+
+// TestAPIGPGListKeysWithKeyringParam tests the HTTP endpoint with custom keyring parameter
+func (s *GPGSuite) TestAPIGPGListKeysWithKeyringParam(c *C) {
+	argFile, err := os.CreateTemp("", "aptly-gpg-args")
+	c.Assert(err, IsNil)
+	_ = argFile.Close()
+	defer func() { _ = os.Remove(argFile.Name()) }()
+
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
+		"cat <<'EOF'\n" +
+		"pub:u:4096:1:8B48AD6246925553:1611864000:::::\n" +
+		"fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:\n" +
+		"EOF\n" +
+		"exit 0\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		response, reqErr := s.HTTPRequest("GET", "/api/gpg/keys?keyring=/custom.gpg", nil)
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		argBytes, readErr := os.ReadFile(argFile.Name())
+		c.Assert(readErr, IsNil)
+		c.Check(string(argBytes), Matches, `(?s).*--keyring\ncustom\.gpg\n.*`)
+	})
+}
+
+// TestAPIGPGListKeysResponseFormat tests that the response has the correct structure
+func (s *GPGSuite) TestAPIGPGListKeysResponseFormat(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, `pub:f:4096:1:A1B2C3D4E5F67890:1611864000:::::
+uid:f::::1611864000::1234567890::Jane Smith <jane@example.com>::::::::::0:
+fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`, "", ""), func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		var result gpgKeyListResponse
+		err = json.NewDecoder(response.Body).Decode(&result)
+		c.Assert(err, IsNil)
+		c.Assert(result.Keys, HasLen, 1)
+		c.Check(result.Keys[0].KeyID, Equals, "A1B2C3D4E5F67890")
+		c.Check(result.Keys[0].Validity, Equals, "f")
+		c.Check(result.Keys[0].CreatedAt, Equals, "1611864000")
+	})
+}
+
+// TestParseGPGOutputEdgeCaseUIDWithoutFields tests UID record with missing fields
+func (s *GPGSuite) TestParseGPGOutputEdgeCaseUIDWithoutFields(c *C) {
+	// UID record with fewer than 10 fields
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should not have user ID since it's in field 9 and this record is too short
+	c.Check(keys[0].UserIDs, HasLen, 0)
+}
+
+// TestParseGPGOutputFingerprintWithoutCurrentKey tests FPR record appearing before any PUB
+func (s *GPGSuite) TestParseGPGOutputFingerprintWithoutCurrentKey(c *C) {
+	// FPR record without a preceding PUB (should be ignored)
+	output := `fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should only have one key with the correct fingerprint
+	c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+}
+
+// TestParseGPGOutputComplexRealWorldExample tests real-world-like GPG output
+func (s *GPGSuite) TestParseGPGOutputComplexRealWorldExample(c *C) {
+	// Real-world GPG output with multiple keys, UIDs, and other record types (sig, sub)
+	// Note: sub and sig records are skipped as we only care about pub/uid/fpr
+	realWorldOutput := `tru::1:1611864000:0:3:1:5
+pub:u:4096:1:8B48AD6246925553:1611864000:2023-01-15T00:00:00:::::scESC:::::::23::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+uid:u::::1611864100::1234567891::John Doe <john@work.com>::::::::::0:
+pub:f:2048:1:1234567890123456:1580592000:2022-12-31T00:00:00::u:::scESC:::::::23::0:
+fpr:::::::::F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9:
+uid:f::::1580592000::0987654321::Maintainer Key <maint@example.com>::::::::::0:`
+
+	keys := parseGPGOutput(realWorldOutput)
+	c.Check(keys, HasLen, 2)
+
+	// First key should have 2 UIDs
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+	c.Check(keys[0].UserIDs, HasLen, 2)
+	c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+
+	// Second key should have 1 UID
+	c.Check(keys[1].KeyID, Equals, "1234567890123456")
+	c.Check(keys[1].UserIDs, HasLen, 1)
+	c.Check(keys[1].Fingerprint, Equals, "F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9")
+}
+
+// TestParseGPGOutputConsecutiveEmptyUIDs tests handling of consecutive empty user ID fields
+func (s *GPGSuite) TestParseGPGOutputConsecutiveEmptyUIDs(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890:::::::::::0:
+uid:u::::1611864000::1234567891::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should skip empty UID but include the non-empty one
+	c.Check(keys[0].UserIDs, HasLen, 1)
+	c.Check(keys[0].UserIDs[0], Equals, "John Doe <john@example.com>")
+}
+
+// TestGPGDeleteKeyParamsValidation tests gpgDeleteKeyParams validation
+func (s *GPGSuite) TestGPGDeleteKeyParamsValidation(c *C) {
+	// This is a unit test that validates parameter structure (no HTTP needed)
+	params := gpgDeleteKeyParams{
+		Keyring: "custom.gpg",
+		GpgKeyID: "8B48AD6246925553",
+	}
+	c.Check(params.Keyring, Equals, "custom.gpg")
+	c.Check(params.GpgKeyID, Equals, "8B48AD6246925553")
+}
+
+// TestAPIGPGDeleteKeyMissingKeyID tests delete with missing key ID parameter
+func (s *GPGSuite) TestAPIGPGDeleteKeyMissingKeyID(c *C) {
+	body, err := json.Marshal(map[string]string{
+		"Keyring": "trustedkeys.gpg",
+		// GpgKeyID is missing
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+}
+
+// TestAPIGPGDeleteKeyInvalidJSON tests delete with invalid JSON request
+func (s *GPGSuite) TestAPIGPGDeleteKeyInvalidJSON(c *C) {
+	response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader([]byte("invalid json")))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+}
+
+// TestAPIGPGDeleteKeySuccess tests successful key deletion
+func (s *GPGSuite) TestAPIGPGDeleteKeySuccess(c *C) {
+	argFile, err := os.CreateTemp("", "aptly-gpg-delete-args")
+	c.Assert(err, IsNil)
+	_ = argFile.Close()
+	defer func() { _ = os.Remove(argFile.Name()) }()
+
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--delete-keys'; then\n" +
+		"echo 'deleted'\n" +
+		"exit 0\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		body, marshalErr := json.Marshal(gpgDeleteKeyParams{
+			Keyring: "/trustedkeys.gpg",
+			GpgKeyID: "8B48AD6246925553",
+		})
+		c.Assert(marshalErr, IsNil)
+
+		response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 200)
+		c.Check(response.Body.String(), Matches, `"deleted\\n"`)
+
+		argBytes, readErr := os.ReadFile(argFile.Name())
+		c.Assert(readErr, IsNil)
+		argText := string(argBytes)
+		c.Check(argText, Matches, `(?s).*--batch\n--yes\n.*`)
+		c.Check(argText, Matches, `(?s).*--keyring\n/trustedkeys\.gpg\n.*`)
+		c.Check(argText, Matches, `(?s).*--delete-keys\n8B48AD6246925553\n.*`)
+	})
+}
+
+// TestAPIGPGListKeysCommandFailure tests list error propagation from gpg
+func (s *GPGSuite) TestAPIGPGListKeysCommandFailure(c *C) {
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
+		"echo 'keyring missing'\n" +
+		"exit 1\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 400)
+		c.Check(response.Body.String(), Matches, `(?s).*failed to list keys: keyring missing.*`)
+	})
+}
+
+// TestAPIGPGDeleteKeyCommandFailure tests delete error propagation from gpg
+func (s *GPGSuite) TestAPIGPGDeleteKeyCommandFailure(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, "", "", "delete failed"), func(_ string) {
+		body, err := json.Marshal(gpgDeleteKeyParams{
+			Keyring: "trustedkeys.gpg",
+			GpgKeyID: "8B48AD6246925553",
+		})
+		c.Assert(err, IsNil)
+
+		response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 400)
+		c.Check(response.Body.String(), Matches, `(?s).*failed to delete key: delete failed.*`)
+	})
+}

api/graph.go

@@ -12,7 +12,27 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+// @Summary Graph Output
+// @Description **Generate dependency graph**
+// @Description
+// @Description Command graph generates graph of dependencies:
+// @Description
+// @Description * between snapshots and mirrors (what mirror was used to create each snapshot)
+// @Description * between snapshots and local repos (what local repo was used to create snapshot)
+// @Description * between snapshots (pulling, merging, etc.)
+// @Description * between snapshots, local repos and published repositories (how snapshots were published).
+// @Description
+// @Description Graph is rendered to PNG file using graphviz package.
+// @Description
+// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical`
+// @Tags Status
+// @Produce image/png, image/svg+xml
+// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg."
+// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout."
+// @Success 200 {object} []byte "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/graph.{ext} [get]
 func apiGraph(c *gin.Context) {
 	var (
 		err    error
@@ -43,25 +63,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/metrics.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+	apiFilesUploadedCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_files_uploaded_total",
+			Help: "Total number of uploaded files labeled by upload directory.",
+		},
+		[]string{"directory"},
+	)
+	apiReposPackageCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_repos_package_count",
+			Help: "Current number of published packages labeled by source, distribution and component.",
+		},
+		[]string{"source", "distribution", "component"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
+
+func countPackagesByRepos() {
+	err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if err != nil {
+			msg := fmt.Sprintf(
+				"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
+				err, repo.StoragePrefix(), repo.Distribution, repo.Components())
+			log.Warn().Msg(msg)
+			return err
+		}
+
+		components := repo.Components()
+		for _, c := range components {
+			count := float64(len(repo.RefList(c).Refs))
+			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
+		log.Warn().Msg(msg)
+	}
+}

api/middleware.go

@@ -9,59 +9,36 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-)
-
-var (
-	apiRequestsInFlightGauge = promauto.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "aptly_api_http_requests_in_flight",
-			Help: "Number of concurrent HTTP api requests currently handled.",
-		},
-		[]string{"method", "path"},
-	)
-	apiRequestsTotalCounter = promauto.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "aptly_api_http_requests_total",
-			Help: "Total number of api requests.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_size_bytes",
-			Help: "Api HTTP request size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiResponseSizeSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_response_size_bytes",
-			Help: "Api HTTP response size in bytes.",
-		},
-		[]string{"code", "method", "path"},
-	)
-	apiRequestsDurationSummary = promauto.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "aptly_api_http_request_duration_seconds",
-			Help: "Duration of api requests in seconds.",
-		},
-		[]string{"code", "method", "path"},
-	)
+	"github.com/rs/zerolog/log"
 )
 
 // Only use base path as label value (e.g.: /api/repos) because of time series cardinality
 // See https://prometheus.io/docs/practices/naming/#labels
 func getBasePath(c *gin.Context) string {
-	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+	segment0, err := getURLSegment(c.Request.URL.Path, 0)
+	if err != nil {
+		return "/"
+	}
+	segment1, err := getURLSegment(c.Request.URL.Path, 1)
+	if err != nil {
+		return *segment0
+	}
+
+	return *segment0 + *segment1
 }
 
-func getURLSegment(url string, idx int) string {
-	var urlSegments = strings.Split(url, "/")
-
+func getURLSegment(url string, idx int) (*string, error) {
+	urlSegments := strings.Split(url, "/")
 	// Remove segment at index 0 because it's an empty string
-	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
-	return fmt.Sprintf("/%s", segmentAtIndex)
+	urlSegments = urlSegments[1:cap(urlSegments)]
+
+	if len(urlSegments) <= idx {
+		return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
+	}
+
+	segmentAtIndex := urlSegments[idx]
+	s := fmt.Sprintf("/%s", segmentAtIndex)
+	return &s, nil
 }
 
 func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
@@ -101,3 +78,39 @@ func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Co
 		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
 	}
 }
+
+// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
+// logs that include error messages if there are any.
+func JSONLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
+
+		// Process request
+		c.Next()
+
+		ts := time.Now()
+		if raw != "" {
+			path = path + "?" + raw
+		}
+
+		errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
+		l := log.With().Str("remote", c.ClientIP()).Logger().
+			With().Str("method", c.Request.Method).Logger().
+			With().Str("path", path).Logger().
+			With().Str("protocol", c.Request.Proto).Logger().
+			With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
+			With().Str("latency", ts.Sub(start).String()).Logger().
+			With().Str("agent", c.Request.UserAgent()).Logger()
+
+		if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
+			l.Warn().Msg(errorMessage)
+		} else if c.Writer.Status() >= 500 {
+			l.Error().Msg(errorMessage)
+		} else {
+			l.Info().Msg(errorMessage)
+		}
+	}
+}

api/middleware_test.go

@@ -0,0 +1,255 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MiddlewareSuite struct {
+	router    http.Handler
+	context   *gin.Context
+	logReader *os.File
+	logWriter *os.File
+}
+
+var _ = Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) SetUpTest(c *C) {
+	r, w, err := os.Pipe()
+	c.Assert(err, IsNil)
+
+	utils.SetupJSONLogger("debug", w)
+	mw := JSONLogger()
+
+	router := gin.New()
+	router.UseRawPath = true
+	router.Use(mw)
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	root := router.Group("/api")
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	root.GET("/ready", apiReady(isReady))
+	root.GET("/healthy", apiHealthy)
+
+	s.router = router
+	s.logReader = r
+	s.logWriter = w
+}
+
+func (s *MiddlewareSuite) TearDownTest(c *C) {
+	s.router = nil
+	s.context = nil
+	s.logReader = nil
+	s.logWriter = nil
+}
+
+func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
+	recorder := httptest.NewRecorder()
+	s.context, _ = gin.CreateTestContext(recorder)
+	req, _ := http.NewRequestWithContext(s.context, method, url, body)
+	s.context.Request = req
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		_, _ = io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	_ = s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "warn")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["method"]; ok {
+		c.Check(val, Equals, "GET")
+	} else {
+		c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["path"]; ok {
+		c.Check(val, Equals, "/")
+	} else {
+		c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["protocol"]; ok {
+		c.Check(val, Equals, "HTTP/1.1")
+	} else {
+		c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["code"]; ok {
+		c.Check(val, Equals, "404")
+	} else {
+		c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["remote"]; !ok {
+		c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["latency"]; !ok {
+		c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["agent"]; !ok {
+		c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["time"]; !ok {
+		c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		_, _ = io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
+	_ = s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		_, _ = io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
+	_ = s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "error")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		_, _ = io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
+	_ = s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	fmt.Println(capturedOutput)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestGetBasePath(c *C) {
+	s.HTTPRequest(http.MethodGet, "", nil)
+	path := getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/api", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api")
+
+	s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api/repos")
+}
+
+func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
+	url := "/"
+	segment, err := getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api/repos/testRepo"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	segment, err = getURLSegment(url, 1)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/repos")
+}

api/mirror.go

@@ -2,8 +2,8 @@ package api
 
 import (
 	"fmt"
-	"log"
 	"net/http"
+	"os"
 	"sort"
 	"strings"
 	"sync"
@@ -14,19 +14,16 @@ import (
 	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
 )
 
-func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
-	if ignoreSignatures {
-		return nil, nil
-	}
-
+func getVerifier(keyRings []string) (pgp.Verifier, error) {
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(false)
 	if err != nil {
 		return nil, err
 	}
@@ -34,38 +31,110 @@ func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error)
 	return verifier, nil
 }
 
-// GET /api/mirrors
+// stringSlicesEqual compares two string slices for equality (order matters)
+func stringSlicesEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// uniqueStrings returns a new slice with only unique strings from the input, sorted
+func uniqueStrings(input []string) []string {
+	if len(input) == 0 {
+		return input
+	}
+	seen := make(map[string]struct{}, len(input))
+	result := make([]string, 0, len(input))
+	for _, s := range input {
+		if _, ok := seen[s]; !ok {
+			seen[s] = struct{}{}
+			result = append(result, s)
+		}
+	}
+	sort.Strings(result)
+	return result
+}
+
+// @Summary List Mirrors
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {array} remoteRepoResponse
+// @Router /api/mirrors [get]
 func apiMirrorsList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
-	result := []*deb.RemoteRepo{}
-	collection.ForEach(func(repo *deb.RemoteRepo) error {
-		result = append(result, repo)
+	result := []remoteRepoResponse{}
+	err := collection.ForEach(func(repo *deb.RemoteRepo) error {
+		err := collection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newRemoteRepoResponse(repo))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
+	}
 
 	c.JSON(200, result)
 }
 
-// POST /api/mirrors
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to mirror AppStream (DEP-11) metadata
+	DownloadAppStream bool `                 json:"DownloadAppStream"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
+// @Summary Create Mirror
+// @Description **Create a mirror of a remote repository**
+// @Tags Mirrors
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/mirrors [post]
 func apiMirrorsCreate(c *gin.Context) {
 	var err error
-	var b struct {
-		Name               string `binding:"required"`
-		ArchiveURL         string `binding:"required"`
-		Distribution       string
-		Filter             string
-		Components         []string
-		Architectures      []string
-		Keyrings           []string
-		DownloadSources    bool
-		DownloadUdebs      bool
-		DownloadInstaller  bool
-		FilterWithDeps     bool
-		SkipComponentCheck bool
-		IgnoreSignatures   bool
-	}
+	var b mirrorCreateParams
 
 	b.DownloadSources = context.Config().DownloadSourcePackages
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -81,7 +150,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
 		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 	}
@@ -89,48 +158,60 @@ func apiMirrorsCreate(c *gin.Context) {
 	if b.Filter != "" {
 		_, err = query.Parse(b.Filter)
 		if err != nil {
-			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 			return
 		}
 	}
 
 	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
-		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream)
 
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
 		return
 	}
 
 	repo.Filter = b.Filter
 	repo.FilterWithDeps = b.FilterWithDeps
 	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.SkipArchitectureCheck = b.SkipArchitectureCheck
 	repo.DownloadSources = b.DownloadSources
 	repo.DownloadUdebs = b.DownloadUdebs
 
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
 	downloader := context.NewDownloader(nil)
-	err = repo.Fetch(downloader, verifier)
+	err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
 		return
 	}
 
 	err = collection.Add(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
 		return
 	}
 
 	c.JSON(201, repo)
 }
 
-// DELETE /api/mirrors/:name
+// @Summary Delete Mirror
+// @Description **Delete a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param force query int true "force: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
+// @Failure 500 {object} Error "Unable to delete"
+// @Router /api/mirrors/{name} [delete]
 func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
@@ -141,13 +222,13 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
@@ -157,7 +238,7 @@ func apiMirrorsDrop(c *gin.Context) {
 			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
@@ -169,7 +250,15 @@ func apiMirrorsDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/mirrors/:name
+// @Summary Get Mirror Info
+// @Description **Get mirror information by name**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [get]
 func apiMirrorsShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -177,19 +266,31 @@ func apiMirrorsShow(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
 	}
 
 	c.JSON(200, repo)
 }
 
-// GET /api/mirrors/:name/packages
+// @Summary List Mirror Packages
+// @Description **Get a list of packages from a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Produce json
+// @Success 200 {array} deb.Package "List of Packages"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name}/packages [get]
 func apiMirrorsPackages(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
@@ -197,17 +298,17 @@ func apiMirrorsPackages(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
 
 	if repo.LastDownloadDate.IsZero() {
-		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
 		return
 	}
 
@@ -216,7 +317,7 @@ func apiMirrorsPackages(c *gin.Context) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -224,7 +325,7 @@ func apiMirrorsPackages(c *gin.Context) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.AbortWithError(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -241,22 +342,26 @@ func apiMirrorsPackages(c *gin.Context) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 		}
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -267,49 +372,179 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 }
 
-// PUT /api/mirrors/:name
+type mirrorEditParams struct {
+	// Package query that is applied to mirror packages
+	Filter *string `         json:"Filter" example:"xserver-xorg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps *bool `   json:"FilterWithDeps"`
+	// Set "true" to mirror installer files
+	DownloadInstaller *bool `json:"DownloadInstaller"`
+	// Set "true" to mirror source packages
+	DownloadSources *bool `  json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs *bool `    json:"DownloadUdebs"`
+	// URL of the archive to mirror
+	ArchiveURL *string `     json:"ArchiveURL"     example:"http://deb.debian.org/debian"`
+	// Comma separated list of architectures
+	Architectures *[]string `json:"Architectures"  example:"amd64"`
+	// Gpg keyring(s) for verifying Release file if a mirror update is required.
+	Keyrings []string `      json:"Keyrings"       example:"trustedkeys.gpg"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures *bool ` json:"IgnoreSignatures"`
+}
+
+// @Summary Edit Mirror
+// @Description **Edit mirror config**
+// @Tags Mirrors
+// @Param name path string true "mirror name to edit"
+// @Consume json
+// @Param request body mirrorEditParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo "Mirror was edited successfully"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 409 {object} Error "Aptly db locked"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [post]
+func apiMirrorsEdit(c *gin.Context) {
+	var (
+		err  error
+		b    mirrorEditParams
+		repo *deb.RemoteRepo
+	)
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err = collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		AbortWithJSONError(c, 409, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
+
+	if b.Filter != nil {
+		repo.Filter = *b.Filter
+	}
+	if b.FilterWithDeps != nil {
+		repo.FilterWithDeps = *b.FilterWithDeps
+	}
+	if b.DownloadInstaller != nil {
+		repo.DownloadInstaller = *b.DownloadInstaller
+	}
+	if b.DownloadSources != nil {
+		repo.DownloadSources = *b.DownloadSources
+	}
+	if b.DownloadUdebs != nil {
+		repo.DownloadUdebs = *b.DownloadUdebs
+	}
+	if b.ArchiveURL != nil && *b.ArchiveURL != repo.ArchiveRoot {
+		repo.SetArchiveRoot(*b.ArchiveURL)
+		fetchMirror = true
+	}
+	if b.Architectures != nil {
+		uniqueArchitectures := uniqueStrings(*b.Architectures)
+		if !stringSlicesEqual(uniqueArchitectures, uniqueStrings(repo.Architectures)) {
+			repo.Architectures = uniqueArchitectures
+			fetchMirror = true
+		}
+	}
+	if b.IgnoreSignatures != nil {
+		ignoreSignatures = *b.IgnoreSignatures
+	}
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to edit: flat mirrors don't support udebs"))
+		return
+	}
+
+	if fetchMirror {
+		verifier, err := getVerifier(b.Keyrings)
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+			return
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
+			return
+		}
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+	// Set "true" to download only the latest version per package/architecture
+	LatestOnly bool `             json:"LatestOnly"`
+}
+
+// @Summary Update Mirror
+// @Description **Update Mirror and download packages**
+// @Tags Mirrors
+// @Param name path string true "mirror name to update"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [put]
 func apiMirrorsUpdate(c *gin.Context) {
 	var (
 		err    error
 		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
 	)
 
-	var b struct {
-		Name                 string
-		ArchiveURL           string
-		Filter               string
-		Architectures        []string
-		Components           []string
-		Keyrings             []string
-		FilterWithDeps       bool
-		DownloadSources      bool
-		DownloadUdebs        bool
-		SkipComponentCheck   bool
-		IgnoreChecksums      bool
-		IgnoreSignatures     bool
-		ForceUpdate          bool
-		SkipExistingPackages bool
-	}
-
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
 	remote, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	b.Name = remote.Name
-	b.DownloadUdebs = remote.DownloadUdebs
-	b.DownloadSources = remote.DownloadSources
-	b.SkipComponentCheck = remote.SkipComponentCheck
-	b.FilterWithDeps = remote.FilterWithDeps
-	b.Filter = remote.Filter
-	b.Architectures = remote.Architectures
-	b.Components = remote.Components
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
-	log.Printf("%s: Starting mirror update\n", b.Name)
+	log.Info().Msgf("%s: Starting mirror update", b.Name)
 
 	if c.Bind(&b) != nil {
 		return
@@ -318,34 +553,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
-			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
 			return
 		}
 	}
 
-	if b.DownloadUdebs != remote.DownloadUdebs {
-		if remote.IsFlat() && b.DownloadUdebs {
-			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
-			return
-		}
-	}
-
-	if b.ArchiveURL != "" {
-		remote.SetArchiveRoot(b.ArchiveURL)
-	}
-
-	remote.Name = b.Name
-	remote.DownloadUdebs = b.DownloadUdebs
-	remote.DownloadSources = b.DownloadSources
-	remote.SkipComponentCheck = b.SkipComponentCheck
-	remote.FilterWithDeps = b.FilterWithDeps
-	remote.Filter = b.Filter
-	remote.Architectures = b.Architectures
-	remote.Components = b.Components
-
-	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
 		return
 	}
 
@@ -353,7 +568,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier)
+		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -365,11 +580,19 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		if remote.DownloadAppStream && !remote.IsFlat() {
+			err = remote.DownloadAppStreamFiles(out, downloader,
+				context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		if remote.Filter != "" {
 			var filterQuery deb.PackageQuery
 
@@ -385,7 +608,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -395,7 +618,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				collection.Update(remote)
+				_ = collection.Update(remote)
 			}
 		}()
 
@@ -458,7 +681,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}()
 
-		log.Printf("%s: Spawning background processes...\n", b.Name)
+		log.Info().Msgf("%s: Spawning background processes...", b.Name)
 		var wg sync.WaitGroup
 		for i := 0; i < context.Config().DownloadConcurrency; i++ {
 			wg.Add(1)
@@ -476,7 +699,16 @@ func apiMirrorsUpdate(c *gin.Context) {
 						var e error
 
 						// provision download location
-						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+							task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+						} else {
+							var file *os.File
+							file, e = os.CreateTemp("", task.File.Filename)
+							if e == nil {
+								task.TempDownPath = file.Name()
+								_ = file.Close()
+							}
+						}
 						if e != nil {
 							pushError(e)
 							continue
@@ -494,6 +726,20 @@ func apiMirrorsUpdate(c *gin.Context) {
 							continue
 						}
 
+						// and import it back to the pool
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						if err != nil {
+							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+							pushError(err)
+							continue
+						}
+
+						// update "attached" files if any
+						for _, additionalAtask := range task.Additional {
+							additionalAtask.File.PoolPath = task.File.PoolPath
+							additionalAtask.File.Checksums = task.File.Checksums
+						}
+
 						task.Done = true
 						taskFinished <- task
 					case <-context.Done():
@@ -505,32 +751,22 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		// Wait for all download goroutines to finish
-		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
 		wg.Wait()
-		log.Printf("%s: Background processes finished\n", b.Name)
+		log.Info().Msgf("%s: Background processes finished", b.Name)
 		close(taskFinished)
 
-		for idx := range queue {
+		defer func() {
+			for _, task := range queue {
+				if task.TempDownPath == "" {
+					continue
+				}
 
-			atask := &queue[idx]
-
-			if !atask.Done {
-				// download not finished yet
-				continue
+				if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+					fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+				}
 			}
-
-			// and import it back to the pool
-			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
-			}
-
-			// update "attached" files if any
-			for _, additionalAtask := range atask.Additional {
-				additionalAtask.File.PoolPath = atask.File.PoolPath
-				additionalAtask.File.Checksums = atask.File.Checksums
-			}
-		}
+		}()
 
 		select {
 		case <-context.Done():
@@ -539,18 +775,18 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		if len(errors) > 0 {
-			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 		}
 
-		log.Printf("%s: Finalizing download\n", b.Name)
-		remote.FinalizeDownload(collectionFactory, out)
+		log.Info().Msgf("%s: Finalizing download...", b.Name)
+		_ = remote.FinalizeDownload(collectionFactory, out)
 		err = collectionFactory.RemoteRepoCollection().Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		log.Info().Msgf("%s: Mirror updated successfully", b.Name)
 		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
 	})
 }

api/mirror_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/gin-gonic/gin"
 	. "gopkg.in/check.v1"
 )
 
 type MirrorSuite struct {
-	ApiSuite
+	APISuite
 }
 
 var _ = Suite(&MirrorSuite{})
@@ -17,7 +18,10 @@ var _ = Suite(&MirrorSuite{})
 func (s *MirrorSuite) TestGetMirrors(c *C) {
 	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "[]")
+
+	var mirrors []map[string]interface{}
+	err := json.Unmarshal(response.Body.Bytes(), &mirrors)
+	c.Assert(err, IsNil)
 }
 
 func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
@@ -26,6 +30,21 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
 	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
 }
 
+func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name":              "test-flat-appstream",
+		"ArchiveURL":        "http://example.com/repo/",
+		"Distribution":      "./",
+		"DownloadAppStream":  true,
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
+}
+
 func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.ExpectFailure("Need to mock downloads")
 	body, err := json.Marshal(gin.H{
@@ -38,3 +57,49 @@ func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.Check(response.Code, Equals, 400)
 	c.Check(response.Body.String(), Equals, "")
 }
+
+func (s *MirrorSuite) TestGetMirrorsIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().RemoteRepoCollection()
+
+	repo, err := deb.NewRemoteRepo("count-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
+	c.Assert(err, IsNil)
+
+	err = collection.Add(repo)
+	c.Assert(err, IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), makePackageRefList(c).Encode())
+
+	response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var mirrors []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &mirrors)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, mirror := range mirrors {
+		if mirror["Name"] == "count-mirror" {
+			found = true
+			value, ok := mirror["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *MirrorSuite) TestGetMirrorsReturns500OnCorruptRefList(c *C) {
+	collection := s.context.NewCollectionFactory().RemoteRepoCollection()
+
+	repo, err := deb.NewRemoteRepo("broken-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
+	c.Assert(err, IsNil)
+	c.Assert(collection.Add(repo), IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
+
+	response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*unable to show:.*")
+}

api/package_count_response.go

@@ -0,0 +1,30 @@
+package api
+
+import "github.com/aptly-dev/aptly/deb"
+
+type remoteRepoResponse struct {
+	*deb.RemoteRepo
+	NumPackages int `json:"NumPackages"`
+}
+
+type localRepoResponse struct {
+	*deb.LocalRepo
+	NumPackages int `json:"NumPackages"`
+}
+
+type snapshotResponse struct {
+	*deb.Snapshot
+	NumPackages int `json:"NumPackages"`
+}
+
+func newRemoteRepoResponse(repo *deb.RemoteRepo) remoteRepoResponse {
+	return remoteRepoResponse{RemoteRepo: repo, NumPackages: repo.NumPackages()}
+}
+
+func newLocalRepoResponse(repo *deb.LocalRepo) localRepoResponse {
+	return localRepoResponse{LocalRepo: repo, NumPackages: repo.NumPackages()}
+}
+
+func newSnapshotResponse(snapshot *deb.Snapshot) snapshotResponse {
+	return snapshotResponse{Snapshot: snapshot, NumPackages: snapshot.NumPackages()}
+}

api/package_count_test_helpers_test.go

@@ -0,0 +1,19 @@
+package api
+
+import (
+	"github.com/aptly-dev/aptly/deb"
+	. "gopkg.in/check.v1"
+)
+
+func makePackageRefList(c *C) *deb.PackageRefList {
+	list := deb.NewPackageList()
+	c.Assert(list.Add(&deb.Package{Name: "libcount", Version: "1.0", Architecture: "amd64"}), IsNil)
+	c.Assert(list.Add(&deb.Package{Name: "appcount", Version: "2.0", Architecture: "all"}), IsNil)
+	return deb.NewPackageRefListFromPackageList(list)
+}
+
+func putRawDBValue(c *C, s *APISuite, key []byte, value []byte) {
+	db, err := s.context.Database()
+	c.Assert(err, IsNil)
+	c.Assert(db.Put(key, value), IsNil)
+}

api/packages.go

@@ -1,17 +1,41 @@
 package api
 
 import (
+	_ "github.com/aptly-dev/aptly/deb" // for swagger
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/packages/:key
+// @Summary Get Package Info
+// @Description **Show information about package by package key**
+// @Description Package keys could be obtained from various GET .../packages APIs.
+// @Tags Packages
+// @Produce json
+// @Param key path string true "package key (unique package identifier)"
+// @Success 200 {object} deb.Package "OK"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/packages/{key} [get]
 func apiPackagesShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, p)
 }
+
+// @Summary List Packages
+// @Description **Get list of packages**
+// @Tags Packages
+// @Consume  json
+// @Produce  json
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Success 200 {array} string "List of packages"
+// @Router /api/packages [get]
+func apiPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PackageCollection()
+	showPackages(c, collection.AllPackageRefs(), collectionFactory)
+}

api/packages_test.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type PackagesSuite struct {
+	APISuite
+}
+
+var _ = Suite(&PackagesSuite{})
+
+func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}

api/repos.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"text/template"
 
@@ -17,28 +18,114 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/repos
+// @Summary Serve HTML Listing
+// @Description If ServeInAPIMode is enabled in aptly config,
+// @Description this endpoint is enabled which returns an HTML listing of each repo that can be browsed
+// @Tags Repos
+// @Produce html
+// @Success 200 {object} string "HTML"
+// @Router /repos [get]
+func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
+		c.Writer.Flush()
+		_, _ = c.Writer.WriteString("<pre>\n")
+		if len(localRepos) == 0 {
+			_, _ = c.Writer.WriteString("<a href=\"-/\">default</a>\n")
+		}
+		for publishPrefix := range localRepos {
+			_, _ = c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
+		}
+		_, _ = c.Writer.WriteString("</pre>")
+		c.Writer.Flush()
+	}
+}
+
+// @Summary Serve Packages
+// @Description If ServeInAPIMode is enabled in aptly config,
+// @Description this api serves a specified package from storage
+// @Tags Repos
+// @Param storage path string true "Storage"
+// @Param pkgPath path string true "Package Path" allowReserved=true
+// @Produce json
+// @Success 200 ""
+// @Router /repos/{storage}/{pkgPath} [get]
+func reposServeInAPIMode(c *gin.Context) {
+	pkgpath := c.Param("pkgPath")
+
+	storage := c.Param("storage")
+	if storage == "-" {
+		storage = ""
+	} else {
+		storage = "filesystem:" + storage
+	}
+
+	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	c.FileFromFS(pkgpath, http.Dir(publicPath))
+}
+
+// @Summary List Repositories
+// @Description **Get list of available repos**
+// @Description Each repo is returned as in “show” API.
+// @Tags Repos
+// @Produce  json
+// @Success 200 {array} localRepoResponse
+// @Router /api/repos [get]
 func apiReposList(c *gin.Context) {
-	result := []*deb.LocalRepo{}
+	result := []localRepoResponse{}
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	collection.ForEach(func(r *deb.LocalRepo) error {
-		result = append(result, r)
+	err := collection.ForEach(func(r *deb.LocalRepo) error {
+		err := collection.LoadComplete(r)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newLocalRepoResponse(r))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
 
 	c.JSON(200, result)
 }
 
-// POST /api/repos
+type repoCreateParams struct {
+	// Name of repository to create
+	Name string `binding:"required"  json:"Name"                 example:"repo1"`
+	// Text describing the repository (optional)
+	Comment string `                 json:"Comment"              example:"this is a repo"`
+	// Default distribution when publishing from this local repo
+	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
+	// Default component when publishing from this local repo
+	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
+	// Snapshot name to create repoitory from (optional)
+	FromSnapshot string `            json:"FromSnapshot"         example:""`
+}
+
+// @Summary Create Repository
+// @Description **Create a local repository**
+// @Description
+// @Description Distribution and component are used as defaults when publishing repo either directly or via snapshot.
+// @Description
+// @Description ```
+// @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Name": "aptly-repo"}' http://localhost:8080/api/repos
+// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
+// @Description ```
+// @Tags Repos
+// @Consume  json
+// @Param request body repoCreateParams true "Parameters"
+// @Produce  json
+// @Success 201 {object} deb.LocalRepo
+// @Failure 404 {object} Error "Source snapshot not found"
+// @Failure 409 {object} Error "Local repo already exists"
+// @Failure 500 {object} Error "Internal error"
+// @Router /api/repos [post]
 func apiReposCreate(c *gin.Context) {
-	var b struct {
-		Name                string `binding:"required"`
-		Comment             string
-		DefaultDistribution string
-		DefaultComponent    string
-	}
+	var b repoCreateParams
 
 	if c.Bind(&b) != nil {
 		return
@@ -49,25 +136,67 @@ func apiReposCreate(c *gin.Context) {
 	repo.DefaultDistribution = b.DefaultDistribution
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.LocalRepoCollection()
-	err := collection.Add(repo)
-	if err != nil {
-		c.AbortWithError(400, err)
+
+	if b.FromSnapshot != "" {
+		var snapshot *deb.Snapshot
+
+		snapshotCollection := collectionFactory.SnapshotCollection()
+
+		snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+		if err != nil {
+			AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
+			return
+		}
+
+		err = snapshotCollection.LoadComplete(snapshot)
+		if err != nil {
+			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
+			return
+		}
+
+		repo.UpdateRefList(snapshot.RefList())
+	}
+
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+
+	if _, err := localRepoCollection.ByName(b.Name); err == nil {
+		AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
 		return
 	}
 
-	c.JSON(201, repo)
-}
-
-// PUT /api/repos/:name
-func apiReposEdit(c *gin.Context) {
-	var b struct {
-		Name                *string
-		Comment             *string
-		DefaultDistribution *string
-		DefaultComponent    *string
+	err := localRepoCollection.Add(repo)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusInternalServerError, err)
+		return
 	}
 
+	c.JSON(http.StatusCreated, repo)
+}
+
+type reposEditParams struct {
+	// Name of repository to modify
+	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
+	// Change Comment of repository
+	Comment *string `                    json:"Comment"              example:"example repo"`
+	// Change Default Distribution for publishing
+	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
+	// Change Devault Component for publishing
+	DefaultComponent *string `        json:"DefaultComponent"     example:""`
+}
+
+// @Summary Update Repository
+// @Description **Update local repository meta information**
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Consume  json
+// @Param request body reposEditParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.LocalRepo "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/repos/{name} [put]
+func apiReposEdit(c *gin.Context) {
+	var b reposEditParams
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -75,17 +204,18 @@ func apiReposEdit(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
-	if b.Name != nil {
+	if b.Name != nil && *b.Name != name {
 		_, err := collection.ByName(*b.Name)
 		if err == nil {
 			// already exists
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, fmt.Errorf("local repo with name %q already exists", *b.Name))
 			return
 		}
 		repo.Name = *b.Name
@@ -102,7 +232,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -110,20 +240,40 @@ func apiReposEdit(c *gin.Context) {
 }
 
 // GET /api/repos/:name
+// @Summary Get Repository Info
+// @Description Returns basic information about local repository.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Produce  json
+// @Success 200 {object} deb.LocalRepo
+// @Failure 404 {object} Error "Repository not found"
+// @Router /api/repos/{name} [get]
 func apiReposShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, repo)
 }
 
-// DELETE /api/repos/:name
+// @Summary Delete Repository
+// @Description Drop/delete a repo
+// @Description Cannot drop repos that are published.
+// @Description Needs force=1 to drop repos used as source by other repos.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param _async query bool false "Run in background and return task object"
+// @Param force query int false "force: 1 to enable"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Repo object"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 404 {object} Error "Repo Conflict"
+// @Router /api/repos/{name} [delete]
 func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	name := c.Params.ByName("name")
@@ -135,13 +285,13 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
@@ -158,31 +308,54 @@ func apiReposDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/repos/:name/packages
+// @Summary List Repo Packages
+// @Description **Return a list of packages present in the repo**
+// @Description
+// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/repos/aptly-repo/packages
+// @Description ["Pi386 aptly 0.8 966561016b44ed80"]
+// @Description ```
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param q query string true "Package query (e.g Name%20(~%20matlab))"
+// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
+// @Param format query string true "Set to 'details' to return extra info about each package"
+// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
+// @Produce json
+// @Success 200 {object} string "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 404 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [get]
 func apiReposPackagesShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	showPackages(c, repo.RefList(), collectionFactory)
 }
 
+type reposPackagesAddDeleteParams struct {
+	// Package Refs
+	PackageRefs []string `binding:"required"     json:"PackageRefs"                 example:""`
+}
+
 // Handler for both add and delete
 func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
-	var b struct {
-		PackageRefs []string
-	}
+	var b reposPackagesAddDeleteParams
 
 	if c.Bind(&b) != nil {
 		return
@@ -193,18 +366,18 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
-		return
-	}
-
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(repo.Key())}
-	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
 		out.Printf("Loading packages...\n")
 		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
 		if err != nil {
@@ -239,7 +412,23 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	})
 }
 
-// POST /repos/:name/packages
+// @Summary Add Packages by Key
+// @Description **Add packages to local repository by package keys.**
+// @Description
+// @Description Any package can be added that is present in the aptly database (from any mirror, snapshot, local repository). This API combined with package list (search) APIs allows one to implement importing, copying, moving packages around.
+// @Description
+// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Consume  json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 400 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [post]
 func apiReposPackagesAdd(c *gin.Context) {
 	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
 		out.Printf("Adding package %s\n", p.Name)
@@ -247,7 +436,21 @@ func apiReposPackagesAdd(c *gin.Context) {
 	})
 }
 
-// DELETE /repos/:name/packages
+// @Summary Delete Packages by Key
+// @Description **Remove packages from local repository by package keys.**
+// @Description
+// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param _async query bool false "Run in background and return task object"
+// @Consume  json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Produce json
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 400 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/packages [delete]
 func apiReposPackagesDelete(c *gin.Context) {
 	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
 		out.Printf("Removing package %s\n", p.Name)
@@ -256,13 +459,43 @@ func apiReposPackagesDelete(c *gin.Context) {
 	})
 }
 
-// POST /repos/:name/file/:dir/:file
+// @Summary Add Uploaded File
+// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
+// @Description Adding same package to local repository is not an error.
+// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string true "Filename"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {string} string "OK"
+// @Failure 400 {object} Error "wrong file"
+// @Failure 404 {object} Error "Repository not found"
+// @Failure 500 {object} Error "Error adding files"
+// @Router /api/repos/{name}/file/{dir}/{file} [post]
 func apiReposPackageFromFile(c *gin.Context) {
 	// redirect all work to dir method
 	apiReposPackageFromDir(c)
 }
 
-// POST /repos/:name/file/:dir
+// @Summary Add Uploaded Directory
+// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
+// @Description Adding same package to local repository is not an error.
+// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory to add"
+// @Consume  json
+// @Param noRemove query string false "when value is set to 1, don’t remove any files"
+// @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce  json
+// @Success 200 {string} string "OK"
+// @Failure 400 {object} Error "wrong file"
+// @Failure 404 {object} Error "Repository not found"
+// @Failure 500 {object} Error "Error adding files"
+// @Router /api/repos/{name}/file/{dir} [post]
 func apiReposPackageFromDir(c *gin.Context) {
 	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
 	noRemove := c.Request.URL.Query().Get("noRemove") == "1"
@@ -271,10 +504,10 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	dirParam := c.Params.ByName("dir")
-	fileParam := c.Params.ByName("file")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -284,13 +517,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	name := c.Params.ByName("name")
 	repo, err := collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
-		return
-	}
-
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -306,7 +533,12 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
 		verifier := context.GetVerifier()
 
 		var (
@@ -355,7 +587,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 			}
 
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -382,13 +614,207 @@ func apiReposPackageFromDir(c *gin.Context) {
 	})
 }
 
-// POST /repos/:name/include/:dir/:file
+type reposCopyPackageParams struct {
+	// Copy also dependencies
+	WithDeps bool `json:"with-deps,omitempty"`
+	// Do not perform operations
+	DryRun bool `json:"dry-run,omitempty"`
+}
+
+// @Summary Copy Package
+// @Description Copies a package from a source to destination repository
+// @Tags Repos
+// @Param name path string true "Destination repo"
+// @Param src path string true "Source repo"
+// @Param file path string true "File/packages to copy"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 422 {object} Error "Unprocessable Entity"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/repos/{name}/copy/{src}/{file} [post]
+func apiReposCopyPackage(c *gin.Context) {
+	dstRepoName := c.Params.ByName("name")
+	srcRepoName := c.Params.ByName("src")
+	fileName := c.Params.ByName("file")
+
+	jsonBody := reposCopyPackageParams{
+		WithDeps: false,
+		DryRun:   false,
+	}
+
+	err := c.Bind(&jsonBody)
+	if err != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
+		return
+	}
+
+	var srcRepo *deb.LocalRepo
+	srcRepo, err = collectionFactory.LocalRepoCollection().ByName(srcRepoName)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
+		return
+	}
+
+	if srcRepo.UUID == dstRepo.UUID {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest and source are identical"))
+		return
+	}
+
+	taskName := fmt.Sprintf("Copy packages from repo %s to repo %s", srcRepoName, dstRepoName)
+	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
+		}
+
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
+		}
+
+		srcRefList := srcRepo.RefList()
+
+		reporter := &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
+		}
+
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
+		}
+
+		srcList.PrepareIndex()
+
+		var architecturesList []string
+
+		if jsonBody.WithDeps {
+			dstList.PrepareIndex()
+
+			// Calculate architectures
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = dstList.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			}
+		}
+
+		// srcList.Filter|FilterWithProgress only accept query list
+		queries := make([]deb.PackageQuery, 1)
+		queries[0], err = query.Parse(fileName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
+		}
+
+		toProcess, err := srcList.Filter(deb.FilterOptions{
+			Queries:           queries,
+			WithDependencies:  jsonBody.WithDeps,
+			Source:            dstList,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+			Progress:          context.Progress(),
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
+		}
+
+		if toProcess.Len() == 0 {
+			return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("no package found for filter: '%s'", fileName)
+		}
+
+		err = toProcess.ForEach(func(p *deb.Package) error {
+			err = dstList.Add(p)
+			if err != nil {
+				return err
+			}
+
+			name := fmt.Sprintf("added %s-%s(%s)", p.Name, p.Version, p.Architecture)
+			reporter.AddedLines = append(reporter.AddedLines, name)
+			return nil
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("error processing dest add: %s", err)
+		}
+
+		if jsonBody.DryRun {
+			reporter.Warning("Changes not saved, as dry run has been requested")
+		} else {
+			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
+
+			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report": reporter,
+		}}, nil
+	})
+}
+
+// @Summary Include File from Directory
+// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string true "File/packages to include"
+// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
+// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
+// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
+// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} string "msg"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/repos/{name}/include/{dir}/{file} [post]
 func apiReposIncludePackageFromFile(c *gin.Context) {
 	// redirect all work to dir method
 	apiReposIncludePackageFromDir(c)
 }
 
-// POST /repos/:name/include/:dir
+type reposIncludePackageFromDirResponse struct {
+	Report      *aptly.RecordingResultReporter
+	FailedFiles []string
+}
+
+// @Summary Include Directory
+// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
+// @Tags Repos
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
+// @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
+// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
+// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} reposIncludePackageFromDirResponse "Response"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/repos/{name}/include/{dir} [post]
 func apiReposIncludePackageFromDir(c *gin.Context) {
 	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
 	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
@@ -404,10 +830,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	var sources []string
 	var taskName string
-	dirParam := c.Params.ByName("dir")
-	fileParam := c.Params.ByName("file")
+	dirParam := utils.SanitizePath(c.Params.ByName("dir"))
+	fileParam := utils.SanitizePath(c.Params.ByName("file"))
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -421,18 +847,18 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
 	if err != nil {
-		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
+		AbortWithJSONError(c, 400, fmt.Errorf("error parsing repo template: %s", err))
 		return
 	}
 
 	var resources []string
-	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+	if len(repoTemplate.Root.Nodes) > 1 {
 		resources = append(resources, task.AllLocalReposResourcesKey)
 	} else {
 		// repo template string is simple text so only use resource key of specific repository
 		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
 		if err != nil {
-			c.AbortWithError(404, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
@@ -440,7 +866,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	}
 	resources = append(resources, sources...)
 
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()
@@ -466,7 +892,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 		if !noRemoveFiles {
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -486,10 +912,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
-			"Report":      reporter,
-			"FailedFiles": failedFiles,
-		}}, nil
-
+		ret := reposIncludePackageFromDirResponse{
+			Report:      reporter,
+			FailedFiles: failedFiles,
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
 	})
 }

api/repos_test.go

@@ -0,0 +1,63 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type ReposSuite struct {
+	APISuite
+}
+
+var _ = Suite(&ReposSuite{})
+
+func (s *ReposSuite) TestGetReposIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().LocalRepoCollection()
+	repo := deb.NewLocalRepo("count-repo-list", "")
+	repo.UpdateRefList(makePackageRefList(c))
+	c.Assert(collection.Add(repo), IsNil)
+
+	response, err := s.HTTPRequest("GET", "/api/repos", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var repos []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &repos)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, repo := range repos {
+		if repo["Name"] == "count-repo-list" {
+			found = true
+			value, ok := repo["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *ReposSuite) TestGetReposReturns500OnCorruptRefList(c *C) {
+	body, err := json.Marshal(gin.H{"Name": "broken-repo-list"})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 201)
+
+	collection := s.context.NewCollectionFactory().LocalRepoCollection()
+	repo, err := collection.ByName("broken-repo-list")
+	c.Assert(err, IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
+
+	response, err = s.HTTPRequest("GET", "/api/repos", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
+}

api/router.go

@@ -2,36 +2,92 @@ package api
 
 import (
 	"net/http"
+	"sync/atomic"
 
+	"github.com/aptly-dev/aptly/aptly"
 	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog/log"
+
+	// _ "github.com/aptly-dev/aptly/docs" // import docs
+	// swaggerFiles "github.com/swaggo/files"
+	// ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
+		countPackagesByRepos()
 		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
 	}
 }
 
+func redirectSwagger(c *gin.Context) {
+	if c.Request.URL.Path == "/docs/index.html" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs/" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	c.Next()
+}
+
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
-	context = c
-
-	router := gin.Default()
-	router.UseRawPath = true
-	router.Use(gin.ErrorLogger())
-
-	if c.Config().EnableMetricsEndpoint {
-		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+	if aptly.EnableDebug {
+		gin.SetMode(gin.DebugMode)
+	} else {
+		gin.SetMode(gin.ReleaseMode)
 	}
 
+	router := gin.New()
+	context = c
+
+	router.UseRawPath = true
+
+	if c.Config().LogFormat == "json" {
+		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
+		router.Use(JSONLogger())
+	} else {
+		router.Use(gin.Logger())
+	}
+
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	// if c.Config().EnableSwaggerEndpoint {
+	// 	router.GET("docs.html", func(c *gin.Context) {
+	// 		c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+	// 	})
+	// 	router.Use(redirectSwagger)
+	// 	url := ginSwagger.URL("/docs/doc.json")
+	// 	router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	// }
+
+	if c.Config().EnableMetricsEndpoint {
+		MetricsCollectorRegistrar.Register(router)
+	}
+
+	if c.Config().ServeInAPIMode {
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
+		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
+	}
+
+	api := router.Group("/api")
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
@@ -40,7 +96,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 		go acquireDatabase()
 
-		router.Use(func(c *gin.Context) {
+		api.Use(func(c *gin.Context) {
 			var err error
 
 			errCh := make(chan error)
@@ -48,7 +104,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			err = <-errCh
 			if err != nil {
-				c.AbortWithError(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -56,7 +112,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 				dbRequests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.AbortWithError(500, err)
+					AbortWithJSONError(c, 500, err)
 				}
 			}()
 
@@ -64,99 +120,122 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		})
 	}
 
-	root := router.Group("/api")
-
 	{
 		if c.Config().EnableMetricsEndpoint {
-			root.GET("/metrics", apiMetricsGet())
+			api.GET("/metrics", apiMetricsGet())
 		}
-		root.GET("/version", apiVersion)
+		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
+
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		api.GET("/ready", apiReady(isReady))
+		api.GET("/healthy", apiHealthy)
 	}
 
 	{
-		root.GET("/repos", apiReposList)
-		root.POST("/repos", apiReposCreate)
-		root.GET("/repos/:name", apiReposShow)
-		root.PUT("/repos/:name", apiReposEdit)
-		root.DELETE("/repos/:name", apiReposDrop)
+		api.GET("/repos", apiReposList)
+		api.POST("/repos", apiReposCreate)
+		api.GET("/repos/:name", apiReposShow)
+		api.PUT("/repos/:name", apiReposEdit)
+		api.DELETE("/repos/:name", apiReposDrop)
 
-		root.GET("/repos/:name/packages", apiReposPackagesShow)
-		root.POST("/repos/:name/packages", apiReposPackagesAdd)
-		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+		api.GET("/repos/:name/packages", apiReposPackagesShow)
+		api.POST("/repos/:name/packages", apiReposPackagesAdd)
+		api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
 
-		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
-		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
 
-		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
-		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+		api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
 
-		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+		api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 
 	{
-		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+		api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
 	{
-		root.GET("/mirrors", apiMirrorsList)
-		root.GET("/mirrors/:name", apiMirrorsShow)
-		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
-		root.POST("/mirrors", apiMirrorsCreate)
-		root.PUT("/mirrors/:name", apiMirrorsUpdate)
-		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+		api.GET("/mirrors", apiMirrorsList)
+		api.GET("/mirrors/:name", apiMirrorsShow)
+		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		api.POST("/mirrors", apiMirrorsCreate)
+		api.POST("/mirrors/:name", apiMirrorsEdit)
+		api.PUT("/mirrors/:name", apiMirrorsUpdate)
+		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
-		root.POST("/gpg/key", apiGPGAddKey)
+		api.GET("/gpg/keys", apiGPGListKeys)
+		api.POST("/gpg/key", apiGPGAddKey)
+		api.DELETE("/gpg/key", apiGPGDeleteKey)
 	}
 
 	{
-		root.GET("/files", apiFilesListDirs)
-		root.POST("/files/:dir", apiFilesUpload)
-		root.GET("/files/:dir", apiFilesListFiles)
-		root.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+		api.GET("/s3", apiS3List)
 	}
 
 	{
-		root.GET("/publish", apiPublishList)
-		root.POST("/publish", apiPublishRepoOrSnapshot)
-		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
-		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
-		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.GET("/files", apiFilesListDirs)
+		api.POST("/files/:dir", apiFilesUpload)
+		api.GET("/files/:dir", apiFilesListFiles)
+		api.DELETE("/files/:dir", apiFilesDeleteDir)
+		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
 	}
 
 	{
-		root.GET("/snapshots", apiSnapshotsList)
-		root.POST("/snapshots", apiSnapshotsCreate)
-		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
-		root.GET("/snapshots/:name", apiSnapshotsShow)
-		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
-		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
-		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.GET("/publish", apiPublishList)
+		api.GET("/publish/:prefix/:distribution", apiPublishShow)
+		api.POST("/publish", apiPublishRepoOrSnapshot)
+		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource)
+		api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges)
+		api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources)
+		api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges)
+		api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource)
+		api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource)
+		api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate)
 	}
 
 	{
-		root.GET("/packages/:key", apiPackagesShow)
+		api.GET("/snapshots", apiSnapshotsList)
+		api.POST("/snapshots", apiSnapshotsCreate)
+		api.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.GET("/snapshots/:name", apiSnapshotsShow)
+		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
 	}
 
 	{
-		root.GET("/graph.:ext", apiGraph)
+		api.GET("/packages/:key", apiPackagesShow)
+		api.GET("/packages", apiPackages)
+	}
+
+	{
+		api.GET("/graph.:ext", apiGraph)
 	}
 	{
-		root.POST("/db/cleanup", apiDbCleanup)
+		api.POST("/db/cleanup", apiDBCleanup)
 	}
 	{
-		root.GET("/tasks", apiTasksList)
-		root.POST("/tasks-clear", apiTasksClear)
-		root.GET("/tasks-wait", apiTasksWait)
-		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
-		root.GET("/tasks/:id/output", apiTasksOutputShow)
-		root.GET("/tasks/:id/detail", apiTasksDetailShow)
-		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
-		root.GET("/tasks/:id", apiTasksShow)
-		root.DELETE("/tasks/:id", apiTasksDelete)
-		root.POST("/tasks-dummy", apiTasksDummy)
+		api.GET("/tasks", apiTasksList)
+		api.POST("/tasks-clear", apiTasksClear)
+		api.GET("/tasks-wait", apiTasksWait)
+		api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		api.GET("/tasks/:id/output", apiTasksOutputShow)
+		api.GET("/tasks/:id/detail", apiTasksDetailShow)
+		api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		api.GET("/tasks/:id", apiTasksShow)
+		api.DELETE("/tasks/:id", apiTasksDelete)
 	}
 
 	return router

api/s3.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary S3 buckets
+// @Description **Get list of S3 buckets**
+// @Description
+// @Description List configured S3 buckets.
+// @Tags Status
+// @Produce json
+// @Success 200 {array} string "List of S3 buckets"
+// @Router /api/s3 [get]
+func apiS3List(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().S3PublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/snapshot.go

@@ -3,15 +3,25 @@ package api
 import (
 	"fmt"
 	"net/http"
+	"sort"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/snapshots
+// @Summary List Snapshots
+// @Description **Get list of snapshots**
+// @Description
+// @Description Each snapshot is returned as in “show” API.
+// @Tags Snapshots
+// @Produce  json
+// @Success 200 {array} snapshotResponse
+// @Router /api/snapshots [get]
 func apiSnapshotsList(c *gin.Context) {
 	SortMethodString := c.Request.URL.Query().Get("sort")
 
@@ -22,28 +32,52 @@ func apiSnapshotsList(c *gin.Context) {
 		SortMethodString = "name"
 	}
 
-	result := []*deb.Snapshot{}
-	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
-		result = append(result, snapshot)
+	result := []snapshotResponse{}
+	err := collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+		err := collection.LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newSnapshotResponse(snapshot))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
 
 	c.JSON(200, result)
 }
 
-// POST /api/mirrors/:name/snapshots/
+type snapshotsCreateFromMirrorParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"     json:"Name"                 example:"snap1"`
+	// Description of snapshot
+	Description string `                json:"Description"`
+}
+
+// @Summary Snapshot Mirror
+// @Description **Create a snapshot of a mirror**
+// @Tags Snapshots
+// @Produce json
+// @Param request body snapshotsCreateFromMirrorParams true "Parameters"
+// @Param name path string true "Mirror name"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 201 {object} deb.Snapshot "Created Snapshot"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Mirror Not Found"
+// @Failure 409 {object} Error "Conflicting snapshot"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/mirrors/{name}/snapshots [post]
 func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	var (
 		err      error
 		repo     *deb.RemoteRepo
 		snapshot *deb.Snapshot
+		b        snapshotsCreateFromMirrorParams
 	)
 
-	var b struct {
-		Name        string `binding:"required"`
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -55,14 +89,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
@@ -90,20 +124,37 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	})
 }
 
-// POST /api/snapshots
+type snapshotsCreateParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"  json:"Name"                 example:"snap2"`
+	// Description of snapshot
+	Description string `             json:"Description"`
+	// List of source snapshots
+	SourceSnapshots []string `       json:"SourceSnapshots"      example:"snap1"`
+	// List of package refs
+	PackageRefs []string `           json:"PackageRefs"          example:""`
+}
+
+// @Summary Snapshot Packages
+// @Description **Create a snapshot from package refs**
+// @Description
+// @Description Refs can be obtained from snapshots, local repos, or mirrors
+// @Tags Snapshots
+// @Param request body snapshotsCreateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 201 {object} deb.Snapshot "Created snapshot"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Source snapshot or package refs not found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots [post]
 func apiSnapshotsCreate(c *gin.Context) {
 	var (
 		err      error
 		snapshot *deb.Snapshot
+		b        snapshotsCreateParams
 	)
 
-	var b struct {
-		Name            string `binding:"required"`
-		Description     string
-		SourceSnapshots []string
-		PackageRefs     []string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -123,20 +174,21 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.AbortWithError(404, err)
-			return
-		}
-
-		err = snapshotCollection.LoadComplete(sources[i])
-		if err != nil {
-			c.AbortWithError(500, err)
+			AbortWithJSONError(c, 404, err)
 			return
 		}
 
 		resources = append(resources, string(sources[i].ResourceKey()))
 	}
 
-	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		for i := range sources {
+			err = snapshotCollection.LoadComplete(sources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+		}
+
 		list := deb.NewPackageList()
 
 		// verify package refs and build package list
@@ -160,23 +212,38 @@ func apiSnapshotsCreate(c *gin.Context) {
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
-		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
 	})
 }
 
-// POST /api/repos/:name/snapshots
+type snapshotsCreateFromRepositoryParams struct {
+	// Name of snapshot to create
+	Name string `binding:"required"               json:"Name"                 example:"snap1"`
+	// Description of snapshot
+	Description string `                          json:"Description"`
+}
+
+// @Summary Snapshot Repository
+// @Description **Create a snapshot of a repository by name**
+// @Tags Snapshots
+// @Consume json
+// @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
+// @Param name path string true "Repository name"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 201 {object} deb.Snapshot "Created snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Failure 404 {object} Error "Repo Not Found"
+// @Router /api/repos/{name}/snapshots [post]
 func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	var (
 		err      error
 		repo     *deb.LocalRepo
 		snapshot *deb.Snapshot
+		b        snapshotsCreateFromRepositoryParams
 	)
 
-	var b struct {
-		Name        string `binding:"required"`
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -188,14 +255,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err := collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -218,18 +285,32 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	})
 }
 
-// PUT /api/snapshots/:name
+type snapshotsUpdateParams struct {
+	// Change Name of snapshot
+	Name string `       json:"Name"  example:"snap2"`
+	// Change Description of snapshot
+	Description string `json:"Description"`
+}
+
+// @Summary Update Snapshot
+// @Description **Update snapshot metadata (Name, Description)**
+// @Tags Snapshots
+// @Param request body snapshotsUpdateParams true "Parameters"
+// @Param name path string true "Snapshot name"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} deb.Snapshot "Updated snapshot object"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 409 {object} Error "Conflicting snapshot"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [put]
 func apiSnapshotsUpdate(c *gin.Context) {
 	var (
 		err      error
 		snapshot *deb.Snapshot
+		b        snapshotsUpdateParams
 	)
 
-	var b struct {
-		Name        string
-		Description string
-	}
-
 	if c.Bind(&b) != nil {
 		return
 	}
@@ -240,13 +321,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
 	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		_, err := collection.ByName(b.Name)
 		if err == nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
@@ -268,27 +349,48 @@ func apiSnapshotsUpdate(c *gin.Context) {
 	})
 }
 
-// GET /api/snapshots/:name
+// @Summary Get Snapshot Info
+// @Description **Query detailed information about a snapshot by name**
+// @Tags Snapshots
+// @Param name path string true "Name of the snapshot"
+// @Produce json
+// @Success 200 {object} deb.Snapshot "msg"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [get]
 func apiSnapshotsShow(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, snapshot)
 }
 
-// DELETE /api/snapshots/:name
+// @Summary Delete Snapshot
+// @Description **Delete snapshot by name**
+// @Description Cannot drop snapshots that are published.
+// @Description Needs force=1 to drop snapshots used as source by other snapshots.
+// @Tags Snapshots
+// @Param name path string true "Snapshot name"
+// @Param force query string false "Force operation"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 ""
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 409 {object} Error "Snapshot in use"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name} [delete]
 func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
@@ -299,13 +401,13 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	resources := []string{string(snapshot.ResourceKey())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		published := publishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
@@ -327,7 +429,19 @@ func apiSnapshotsDrop(c *gin.Context) {
 	})
 }
 
-// GET /api/snapshots/:name/diff/:withSnapshot
+// @Summary Snapshot diff
+// @Description **Return the diff between two snapshots (name & withSnapshot)**
+// @Description Provide `onlyMatching=1` to return only packages present in both snapshots.
+// @Description Otherwise, returns a `left` and `right` result providing packages only in the first and second snapshots
+// @Tags Snapshots
+// @Produce json
+// @Param name path string true "Snapshot name"
+// @Param withSnapshot path string true "Snapshot name to diff against"
+// @Param onlyMatching query string false "Only return packages present in both snapshots"
+// @Success 200 {array} deb.PackageDiff "Package Diff"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name}/diff/{withSnapshot} [get]
 func apiSnapshotsDiff(c *gin.Context) {
 	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
 
@@ -336,32 +450,32 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
@@ -378,22 +492,334 @@ func apiSnapshotsDiff(c *gin.Context) {
 	c.JSON(200, result)
 }
 
-// GET /api/snapshots/:name/packages
+// @Summary List Snapshot Packages
+// @Description **List all packages in snapshot or perform search on snapshot contents and return results**
+// @Description If `q` query parameter is missing, return all packages, otherwise return packages that match q
+// @Tags Snapshots
+// @Produce json
+// @Param name path string true "Snapshot to search"
+// @Param q query string false "Package query (e.g Name%20(~%20matlab))"
+// @Param withDeps query string false "Set to 1 to include dependencies when evaluating package query"
+// @Param format query string false "Set to 'details' to return extra info about each package"
+// @Param maximumVersion query string false "Set to 1 to only return the highest version for each package name"
+// @Success 200 {array} string "Package info"
+// @Failure 404 {object} Error "Snapshot Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/snapshots/{name}/packages [get]
 func apiSnapshotsSearchPackages(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	showPackages(c, snapshot.RefList(), collectionFactory)
 }
+
+type snapshotsMergeParams struct {
+	// List of snapshot names to be merged
+	Sources []string `binding:"required" json:"Sources"     example:"snapshot1"`
+}
+
+// @Summary Snapshot Merge
+// @Description **Merge several source snapshots into a new snapshot**
+// @Description
+// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
+// @Description
+// @Description If only one snapshot is specified, merge copies source into destination.
+// @Tags Snapshots
+// @Consume json
+// @Produce json
+// @Param name path string true "Name of the snapshot to be created"
+// @Param latest query int false "merge only the latest version of each package"
+// @Param no-remove query int false "all versions of packages are preserved during merge"
+// @Param request body snapshotsMergeParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Success 201 {object} deb.Snapshot "Resulting snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/merge [post]
+func apiSnapshotsMerge(c *gin.Context) {
+	var (
+		err      error
+		snapshot *deb.Snapshot
+		body     snapshotsMergeParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if c.Bind(&body) != nil {
+		return
+	}
+
+	if len(body.Sources) < 1 {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("minimum one source snapshot is required"))
+		return
+	}
+
+	latest := c.Request.URL.Query().Get("latest") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+	overrideMatching := !latest && !noRemove
+
+	if noRemove && latest {
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("no-remove and latest are mutually exclusive"))
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	sources := make([]*deb.Snapshot, len(body.Sources))
+	resources := make([]string, len(sources))
+	for i := range body.Sources {
+		sources[i], err = snapshotCollection.ByName(body.Sources[i])
+		if err != nil {
+			AbortWithJSONError(c, http.StatusNotFound, err)
+			return
+		}
+
+		resources[i] = string(sources[i].ResourceKey())
+	}
+
+	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = snapshotCollection.LoadComplete(sources[0])
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		result := sources[0].RefList()
+		for i := 1; i < len(sources); i++ {
+			err = snapshotCollection.LoadComplete(sources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+		}
+
+		if latest {
+			result.FilterLatestRefs()
+		}
+
+		sourceDescription := make([]string, len(sources))
+		for i, s := range sources {
+			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
+		}
+
+		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
+}
+
+type snapshotsPullParams struct {
+	// Source name to be searched for packages and dependencies
+	Source string `binding:"required"      json:"Source"            example:"source-snapshot"`
+	// Name of the snapshot to be created
+	Destination string `binding:"required" json:"Destination"       example:"idestination-snapshot"`
+	// List of package queries (i.e. name of package to be pulled from `Source`)
+	Queries []string `binding:"required"   json:"Queries"           example:"xserver-xorg"`
+	// List of architectures (optional)
+	Architectures []string `               json:"Architectures"     example:"amd64, armhf"`
+}
+
+// @Summary Snapshot Pull
+// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
+// @Description
+// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
+// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
+// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
+// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
+// @Description
+// @Description With flag `no-remove` both package versions would stay in the snapshot.
+// @Description
+// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
+// @Tags Snapshots
+// @Param request body snapshotsPullParams true "Parameters"
+// @Param name path string true "Name of the snapshot to be created"
+// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
+// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
+// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
+// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Consume json
+// @Produce json
+// @Success 200 {object} deb.Snapshot "Resulting Snapshot object"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/snapshots/{name}/pull [post]
+func apiSnapshotsPull(c *gin.Context) {
+	var (
+		err                 error
+		destinationSnapshot *deb.Snapshot
+		body                snapshotsPullParams
+	)
+
+	name := c.Params.ByName("name")
+
+	if err = c.BindJSON(&body); err != nil {
+		AbortWithJSONError(c, http.StatusBadRequest, err)
+		return
+	}
+
+	allMatches := c.Request.URL.Query().Get("all-matches") == "1"
+	dryRun := c.Request.URL.Query().Get("dry-run") == "1"
+	noDeps := c.Request.URL.Query().Get("no-deps") == "1"
+	noRemove := c.Request.URL.Query().Get("no-remove") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+
+	// Load <name> snapshot
+	toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+
+	// Load <Source> snapshot
+	sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+
+	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// convert snapshots to package list
+		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		toPackageList.PrepareIndex()
+		sourcePackageList.PrepareIndex()
+
+		var architecturesList []string
+
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = toPackageList.Architectures(false)
+		}
+
+		architecturesList = append(architecturesList, body.Architectures...)
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
+		var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
+		for _, arch := range architecturesList {
+			archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
+		}
+
+		queries := make([]deb.PackageQuery, len(body.Queries))
+		for i, q := range body.Queries {
+			queries[i], err = query.Parse(q)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// Add architecture filter
+			queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
+		}
+
+		// Filter with dependencies as requested
+		destinationPackageList, err := sourcePackageList.Filter(deb.FilterOptions{
+			Queries:           queries,
+			WithDependencies:  !noDeps,
+			Source:            toPackageList,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+			Progress:          context.Progress(),
+		})
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		destinationPackageList.PrepareIndex()
+
+		removedPackages := []string{}
+		addedPackages := []string{}
+		alreadySeen := map[string]bool{}
+
+		_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+			key := pkg.Architecture + "_" + pkg.Name
+			_, seen := alreadySeen[key]
+
+			// If we haven't seen such name-architecture pair and were instructed to remove, remove it
+			if !noRemove && !seen {
+				// Remove all packages with the same name and architecture
+				packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
+				for _, p := range packageSearchResults {
+					toPackageList.Remove(p)
+					removedPackages = append(removedPackages, p.String())
+				}
+			}
+
+			// If !allMatches, add only first matching name-arch package
+			if !seen || allMatches {
+				_ = toPackageList.Add(pkg)
+				addedPackages = append(addedPackages, pkg.String())
+			}
+
+			alreadySeen[key] = true
+
+			return nil
+		})
+		alreadySeen = nil
+
+		if dryRun {
+			response := struct {
+				AddedPackages   []string `json:"added_packages"`
+				RemovedPackages []string `json:"removed_packages"`
+			}{
+				AddedPackages:   addedPackages,
+				RemovedPackages: removedPackages,
+			}
+
+			return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
+		}
+
+		// Create <destination> snapshot
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+
+		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
+	})
+}

api/snapshot_test.go

@@ -0,0 +1,53 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/aptly-dev/aptly/deb"
+	. "gopkg.in/check.v1"
+)
+
+type SnapshotsSuite struct {
+	APISuite
+}
+
+var _ = Suite(&SnapshotsSuite{})
+
+func (s *SnapshotsSuite) TestGetSnapshotsIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().SnapshotCollection()
+	snapshot := deb.NewSnapshotFromRefList("count-snapshot-list", nil, makePackageRefList(c), "")
+	c.Assert(collection.Add(snapshot), IsNil)
+
+	response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var snapshots []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &snapshots)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, snapshot := range snapshots {
+		if snapshot["Name"] == "count-snapshot-list" {
+			found = true
+			value, ok := snapshot["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *SnapshotsSuite) TestGetSnapshotsReturns500OnCorruptRefList(c *C) {
+	collection := s.context.NewCollectionFactory().SnapshotCollection()
+	snapshot := deb.NewSnapshotFromRefList("broken-snapshot-list", nil, makePackageRefList(c), "")
+	c.Assert(collection.Add(snapshot), IsNil)
+	putRawDBValue(c, &s.APISuite, snapshot.RefKey(), []byte("not-msgpack"))
+
+	response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
+}

api/storage.go

@@ -0,0 +1,45 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Description
+// @Description Units in MiB.
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

api/task.go

@@ -1,155 +1,203 @@
 package api
 
 import (
-	"fmt"
-	"net/http"
 	"strconv"
 
-	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
-// GET /tasks
+// @Summary List Tasks
+// @Description **Get list of available tasks. Each task is returned as in “show” API**
+// @Tags Tasks
+// @Produce json
+// @Success 200 {array} task.Task
+// @Router /api/tasks [get]
 func apiTasksList(c *gin.Context) {
 	list := context.TaskList()
 	c.JSON(200, list.GetTasks())
 }
 
-// POST /tasks-clear
+// @Summary Clear Tasks
+// @Description **Removes finished and failed tasks from internal task list**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-clear [post]
 func apiTasksClear(c *gin.Context) {
 	list := context.TaskList()
 	list.Clear()
 	c.JSON(200, gin.H{})
 }
 
-// GET /tasks-wait
+// @Summary Wait for all Tasks
+// @Description **Waits for and returns when all running tasks are complete**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-wait [get]
 func apiTasksWait(c *gin.Context) {
 	list := context.TaskList()
 	list.Wait()
 	c.JSON(200, gin.H{})
 }
 
-// GET /tasks/:id/wait
+// @Summary Wait for Task
+// @Description **Waits for and returns when given Task ID is complete**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 400 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/wait [get]
 func apiTasksWaitForTaskByID(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	task, err := list.WaitForTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, task)
 }
 
-// GET /tasks/:id
+// @Summary Get Task Info
+// @Description **Return task information for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id} [get]
 func apiTasksShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var task task.Task
 	task, err = list.GetTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, task)
 }
 
-// GET /tasks/:id/output
+// @Summary Get Task Output
+// @Description **Return task output for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task output"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/output [get]
 func apiTasksOutputShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var output string
 	output, err = list.GetTaskOutputByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, output)
 }
 
-// GET /tasks/:id/detail
+// @Summary Get Task Details
+// @Description **Return task detail for a given ID**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task detail"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/detail [get]
 func apiTasksDetailShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var detail interface{}
 	detail, err = list.GetTaskDetailByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, detail)
 }
 
-// GET /tasks/:id/return_value
+// @Summary Get Task Return Value
+// @Description **Return task return value (status code) by given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/tasks/{id}/return_value [get]
 func apiTasksReturnValueShow(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err := list.GetTaskReturnValueByID(int(id))
 	if err != nil {
-		c.AbortWithError(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, output)
 }
 
-// DELETE /tasks/:id
+// @Summary Delete Task
+// @Description **Delete completed task by given ID. Does not stop task execution**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 400 {object} Error "Task in progress or not found"
+// @Router /api/tasks/{id} [delete]
 func apiTasksDelete(c *gin.Context) {
 	list := context.TaskList()
 	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
 	if err != nil {
-		c.AbortWithError(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	var delTask task.Task
 	delTask, err = list.DeleteTaskByID(int(id))
 	if err != nil {
-		c.AbortWithError(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, delTask)
 }
-
-// POST /tasks-dummy
-func apiTasksDummy(c *gin.Context) {
-	resources := []string{"dummy"}
-	taskName := fmt.Sprintf("Dummy task")
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		out.Printf("Dummy task started\n")
-		detail.Store([]int{1, 2, 3})
-		out.Printf("Dummy task finished\n")
-		return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
-	})
-}

api/task_test.go

@@ -1,75 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/aptly-dev/aptly/task"
-
-	. "gopkg.in/check.v1"
-)
-
-type TaskSuite struct {
-	ApiSuite
-}
-
-var _ = Suite(&TaskSuite{})
-
-func (s *TaskSuite) TestTasksDummy(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
-	c.Check(response.Code, Equals, 418)
-	c.Check(response.Body.String(), Equals, "[1,2,3]")
-}
-
-func (s *TaskSuite) TestTasksDummyAsync(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	var t task.Task
-	err := json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.Name, Equals, "Dummy task")
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
-	err = json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.State, Equals, task.SUCCEEDED)
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "[1,2,3]")
-	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
-}
-
-func (s *TaskSuite) TestTaskDelete(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
-	// Give the task time to start
-	time.Sleep(time.Second)
-	response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
-	c.Check(response.Code, Equals, 200)
-}
-
-func (s *TaskSuite) TestTasksClear(c *C) {
-	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
-	c.Check(response.Code, Equals, 202)
-	var t task.Task
-	err := json.Unmarshal(response.Body.Bytes(), &t)
-	c.Assert(err, IsNil)
-	c.Check(t.Name, Equals, "Dummy task")
-	response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
-	c.Check(response.Code, Equals, 200)
-	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
-	c.Check(response.Code, Equals, 200)
-	var ts []task.Task
-	err = json.Unmarshal(response.Body.Bytes(), &ts)
-	c.Assert(err, IsNil)
-	c.Check(len(ts), Equals, 1)
-	c.Check(ts[0].State, Equals, task.SUCCEEDED)
-	response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
-	c.Check(response.Code, Equals, 200)
-	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "null")
-}

aptly-api.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository API
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/api/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
-
-[Install]
-WantedBy=multi-user.target

aptly.service

@@ -1,12 +0,0 @@
-[Unit]
-Description=APT repository server
-After=network.target
-Documentation=man:aptly(1)
-Documentation=https://www.aptly.info/doc/commands/
-
-[Service]
-Type=simple
-ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
-
-[Install]
-WantedBy=multi-user.target

aptly/conf.go

@@ -0,0 +1,4 @@
+package aptly
+
+// AptlyConf holds the default aptly.conf (filled in at link time)
+var AptlyConf []byte

aptly/const.go

@@ -0,0 +1,3 @@
+package aptly
+
+var DistributionFocal = "focal"

aptly/interfaces.go

@@ -35,8 +35,8 @@ type PackagePool interface {
 	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
 	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
 	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
-	// Stat returns Unix stat(2) info
-	Stat(path string) (os.FileInfo, error)
+	// Size returns the size of the given file in bytes.
+	Size(path string) (size int64, err error)
 	// Open returns ReadSeekerCloser to access the file
 	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
@@ -47,6 +47,8 @@ type PackagePool interface {
 
 // LocalPackagePool is implemented by PackagePools residing on the same filesystem
 type LocalPackagePool interface {
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
 	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
 	GenerateTempPath(filename string) (string, error)
 	// Link generates hardlink to destination path
@@ -70,7 +72,7 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file

azure/azure.go

@@ -1,2 +1,129 @@
-// Package azure handles publishing to Azure Storage
 package azure
+
+// Package azure handles publishing to Azure Storage
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+func isBlobNotFound(err error) bool {
+	storageError, ok := err.(azblob.StorageError)
+	return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
+}
+
+type azContext struct {
+	container azblob.ContainerURL
+	prefix    string
+}
+
+func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+	}
+
+	url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
+	if err != nil {
+		return nil, err
+	}
+
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
+	result := &azContext{
+		container: containerURL,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+func (az *azContext) blobPath(path string) string {
+	return filepath.Join(az.prefix, path)
+}
+
+func (az *azContext) blobURL(path string) azblob.BlobURL {
+	return az.container.NewBlobURL(az.blobPath(path))
+}
+
+func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(az.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := az.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
+		}
+
+		marker = listBlob.NextMarker
+
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
+		}
+
+		if progress != nil {
+			time.Sleep(time.Duration(500) * time.Millisecond)
+			progress.AddBar(1)
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
+	}
+
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
+		}
+	}
+
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob.ToBlockBlobURL(),
+		uploadOptions,
+	)
+
+	return err
+}
+
+// String
+func (az *azContext) String() string {
+	return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
+}

azure/package_pool.go

@@ -0,0 +1,218 @@
+package azure
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+type PackagePool struct {
+	az *azContext
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates published storage from Azure storage credentials
+func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagePool{az: azctx}, nil
+}
+
+// String
+func (pool *PackagePool) String() string {
+	return pool.az.String()
+}
+
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
+	hash := checksums.SHA256
+	// Use the same path as the file pool, for compat reasons.
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
+}
+
+func (pool *PackagePool) ensureChecksums(
+	poolPath string,
+	checksumStorage aptly.ChecksumStorage,
+) (*utils.ChecksumInfo, error) {
+	targetChecksums, err := checksumStorage.Get(poolPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		blob := pool.az.blobURL(poolPath)
+		download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+		if err != nil {
+			if isBlobNotFound(err) {
+				return nil, nil
+			}
+
+			return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
+		}
+
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
+		if err != nil {
+			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return targetChecksums, nil
+}
+
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	if progress != nil {
+		progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
+		defer progress.ShutdownBar()
+	}
+
+	paths, _, err := pool.az.internalFilelist("", progress)
+	return paths, err
+}
+
+func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
+	return "", errors.New("Azure package pool does not support legacy paths")
+}
+
+func (pool *PackagePool) Size(path string) (int64, error) {
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	return props.ContentLength(), nil
+}
+
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	blob := pool.az.blobURL(path)
+
+	temp, err := os.CreateTemp("", "blob-download")
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating temporary file for blob download")
+	}
+
+	defer os.Remove(temp.Name())
+
+	err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "error downloading blob at %s", path)
+	}
+
+	return temp, nil
+}
+
+func (pool *PackagePool) Remove(path string) (int64, error) {
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
+	}
+
+	_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	if err != nil {
+		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
+	}
+
+	return props.ContentLength(), nil
+}
+
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		var err error
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	path := pool.buildPoolPath(basename, checksums)
+	blob := pool.az.blobURL(path)
+	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
+	if err != nil {
+		return "", err
+	} else if targetChecksums != nil {
+		// target already exists
+		*checksums = *targetChecksums
+		return path, nil
+	}
+
+	source, err := os.Open(srcPath)
+	if err != nil {
+		return "", err
+	}
+	defer source.Close()
+
+	err = pool.az.putFile(blob, source, checksums.MD5)
+	if err != nil {
+		return "", err
+	}
+
+	if !checksums.Complete() {
+		// need full checksums here
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	err = checksumStorage.Update(path, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	if poolPath == "" {
+		if checksums.SHA256 != "" {
+			poolPath = pool.buildPoolPath(basename, checksums)
+		} else {
+			// No checksums or pool path, so no idea what file to look for.
+			return "", false, nil
+		}
+	}
+
+	size, err := pool.Size(poolPath)
+	if err != nil {
+		return "", false, err
+	} else if size != checksums.Size {
+		return "", false, nil
+	}
+
+	targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
+	if err != nil {
+		return "", false, err
+	} else if targetChecksums == nil {
+		return "", false, nil
+	}
+
+	if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+		checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+		// wrong file?
+		return "", false, nil
+	}
+
+	// fill back checksums
+	*checksums = *targetChecksums
+	return poolPath, true, nil
+}

azure/package_pool_test.go

@@ -0,0 +1,255 @@
+package azure
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type PackagePoolSuite struct {
+	accountName, accountKey, endpoint string
+	pool, prefixedPool                *PackagePool
+	debFile                           string
+	cs                                aptly.ChecksumStorage
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+	cnt := s.pool.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+	c.Assert(err, IsNil)
+
+	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = files.NewMockChecksumStorage()
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
+	})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	var checksum utils.ChecksumInfo
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	// import as different name
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	// double import, should be ok
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with path-relevant checksums already filled in
+	checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
+	path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	checksum := utils.ChecksumInfo{}
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with checksums missing (that aren't needed to find the path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
+		SHA256: "abc",
+		Size:   checksum.Size,
+	}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksums (that aren't needed to find the path)
+	// and no info in checksum storage
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	checksum = utils.ChecksumInfo{Size: 13455}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestSize(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Size("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}

azure/public.go

@@ -2,12 +2,8 @@ package azure
 
 import (
 	"context"
-	"encoding/hex"
 	"fmt"
-	"io"
-	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"time"
@@ -22,7 +18,8 @@ import (
 type PublishedStorage struct {
 	container azblob.ContainerURL
 	prefix    string
-	pathCache map[string]string
+	az        *azContext
+	pathCache map[string]map[string]string
 }
 
 // Check interface
@@ -30,60 +27,23 @@ var (
 	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
 )
 
-func isEmulatorEndpoint(endpoint string) bool {
-	if h, _, err := net.SplitHostPort(endpoint); err == nil {
-		endpoint = h
-	}
-	if endpoint == "localhost" {
-		return true
-	}
-	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
-	// In this case, eliminate the '[' and ']' in the URL.
-	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
-	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
-		endpoint = endpoint[1 : len(endpoint)-1]
-	}
-	return net.ParseIP(endpoint) != nil
-}
-
 // NewPublishedStorage creates published storage from Azure storage credentials
 func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
-	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	if endpoint == "" {
-		endpoint = "blob.core.windows.net"
-	}
-
-	var url *url.URL
-	if isEmulatorEndpoint(endpoint) {
-		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
-	} else {
-		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
-
-	result := &PublishedStorage{
-		container: containerURL,
-		prefix:    prefix,
-	}
-
-	return result, nil
+	return &PublishedStorage{az: azctx}, nil
 }
 
 // String
 func (storage *PublishedStorage) String() string {
-	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+	return storage.az.String()
 }
 
 // MkDir creates directory recursively under public path
-func (storage *PublishedStorage) MkDir(path string) error {
+func (storage *PublishedStorage) MkDir(_ string) error {
 	// no op for Azure
 	return nil
 }
@@ -106,7 +66,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer source.Close()
 
-	err = storage.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -114,45 +74,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	return err
 }
 
-// putFile uploads file-like object to
-func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
-	path = filepath.Join(storage.prefix, path)
-
-	blob := storage.container.NewBlockBlobURL(path)
-
-	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
-		BufferSize: 4 * 1024 * 1024,
-		MaxBuffers: 8,
-	}
-	if len(sourceMD5) > 0 {
-		decodedMD5, err := hex.DecodeString(sourceMD5)
-		if err != nil {
-			return err
-		}
-		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
-			ContentMD5: decodedMD5,
-		}
-	}
-
-	_, err := azblob.UploadStreamToBlockBlob(
-		context.Background(),
-		source,
-		blob,
-		uploadOptions,
-	)
-
-	return err
-}
-
 // RemoveDirs removes directory structure under public path
-func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
+		blob := storage.az.blobURL(filepath.Join(path, filename))
 		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
@@ -164,7 +94,7 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
@@ -174,31 +104,39 @@ func (storage *PublishedStorage) Remove(path string) error {
 
 // LinkFromPool links package file from pool to dist's pool location
 //
-// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, fileName)
-	poolPath := filepath.Join(storage.prefix, relPath)
+	relFilePath := filepath.Join(publishedRelPath, fileName)
+	// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	// FIXME: check how to integrate publishedPrefix:
+	poolPath := storage.az.blobPath(fileName)
 
 	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist("")
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
 		if err != nil {
 			return fmt.Errorf("error caching paths under prefix: %s", err)
 		}
 
-		storage.pathCache = make(map[string]string, len(paths))
+		pathCache = make(map[string]string, len(paths))
 
 		for i := range paths {
-			storage.pathCache[paths[i]] = md5s[i]
+			pathCache[paths[i]] = md5s[i]
 		}
+		storage.pathCache[publishedPrefix] = pathCache
 	}
 
-	destinationMD5, exists := storage.pathCache[relPath]
+	destinationMD5, exists := pathCache[relFilePath]
 	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
@@ -221,9 +159,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	}
 	defer source.Close()
 
-	err = storage.putFile(relPath, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
 	if err == nil {
-		storage.pathCache[relPath] = sourceMD5
+		pathCache[relFilePath] = sourceMD5
 	} else {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}
@@ -231,43 +169,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
 	return err
 }
 
-func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
-	const delimiter = "/"
-	paths = make([]string, 0, 1024)
-	md5s = make([]string, 0, 1024)
-	prefix = filepath.Join(storage.prefix, prefix)
-	if prefix != "" {
-		prefix += delimiter
-	}
-
-	for marker := (azblob.Marker{}); marker.NotDone(); {
-		listBlob, err := storage.container.ListBlobsFlatSegment(
-			context.Background(), marker, azblob.ListBlobsSegmentOptions{
-				Prefix:     prefix,
-				MaxResults: 1000,
-				Details:    azblob.BlobListingDetails{Metadata: true}})
-		if err != nil {
-			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
-		}
-
-		marker = listBlob.NextMarker
-
-		for _, blob := range listBlob.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, blob.Name)
-			} else {
-				paths = append(paths, blob.Name[len(prefix):])
-			}
-			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
-		}
-	}
-
-	return paths, md5s, nil
-}
-
 // Filelist returns list of files under prefix
 func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
-	paths, _, err := storage.internalFilelist(prefix)
+	paths, _, err := storage.az.internalFilelist(prefix, nil)
 	return paths, err
 }
 
@@ -275,8 +179,8 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
 	const leaseDuration = 30
 
-	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
-	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
+	dstBlobURL := storage.az.blobURL(dst)
+	srcBlobURL := storage.az.blobURL(src)
 	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
 	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
 		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
@@ -347,11 +251,10 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		storageError, ok := err.(azblob.StorageError)
-		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+		if isBlobNotFound(err) {
 			return false, nil
 		}
 		return false, err
@@ -364,7 +267,7 @@ func (storage *PublishedStorage) FileExists(path string) (bool, error) {
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	blob := storage.az.blobURL(path)
 	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return "", err

azure/public_test.go

@@ -43,14 +43,14 @@ func randString(n int) string {
 func (s *PublishedStorageSuite) SetUpSuite(c *C) {
 	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
 	if s.accountName == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
 	}
 	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
 	if s.accountKey == "" {
-		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("Please set the following two environment variables to run the Azure storage tests.")
 		println("  1. AZURE_STORAGE_ACCOUNT")
 		println("  2. AZURE_STORAGE_ACCESS_KEY")
 		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
@@ -66,7 +66,7 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-	cnt := s.storage.container
+	cnt := s.storage.az.container
 	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
@@ -75,13 +75,13 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-	cnt := s.storage.container
+	cnt := s.storage.az.container
 	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-	blob := s.storage.container.NewBlobURL(path)
+	blob := s.storage.az.container.NewBlobURL(path)
 	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, IsNil)
 	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
@@ -91,7 +91,7 @@ func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-	_, err := s.storage.container.NewBlobURL(path).GetProperties(
+	_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
 		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, NotNil)
 	storageError, ok := err.(azblob.StorageError)
@@ -104,7 +104,7 @@ func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	_, err := azblob.UploadBufferToBlockBlob(
 		context.Background(),
 		data,
-		s.storage.container.NewBlockBlobURL(path),
+		s.storage.az.container.NewBlockBlobURL(path),
 		azblob.UploadToBlockBlobOptions{
 			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
 				ContentMD5: hash[:],
@@ -129,7 +129,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
 	c.Check(err, IsNil)
 
-	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
 }
 
 func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
@@ -300,45 +300,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 
 	// first link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// duplicate link from pool
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with conflict and force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
 
 	// for prefixed storage:
 	// first link from pool
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
 	c.Check(err, IsNil)
 
 	// 2nd link from pool, providing wrong path for source file
 	//
 	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
-	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
 	// link from pool with nested file name
-	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
 	c.Check(err, IsNil)
 
 	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))

cmd/api_serve.go

@@ -1,11 +1,15 @@
 package cmd
 
 import (
+	stdcontext "context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/aptly-dev/aptly/api"
 	"github.com/aptly-dev/aptly/systemd/activation"
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}
@@ -42,7 +46,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	}
 	if err == nil && len(listeners) == 1 {
 		listener := listeners[0]
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
 		err = http.Serve(listener, api.Router(context))
 		if err != nil {
@@ -55,31 +59,42 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
+	server := http.Server{Handler: api.Router(context)}
+
+	sigchan := make(chan os.Signal, 1)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go (func() {
+		if _, ok := <-sigchan; ok {
+			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
+			context.TaskList().Wait()
+			_ = server.Shutdown(stdcontext.Background())
+		}
+	})()
+	defer close(sigchan)
+
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-		os.Remove(file)
+		_ = os.Remove(file)
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)
 		if err != nil {
 			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
 		}
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 
-		err = http.Serve(listener, api.Router(context))
-		if err != nil {
-			return fmt.Errorf("unable to serve: %s", err)
-		}
-		return nil
+		err = server.Serve(listener)
+	} else {
+		server.Addr = listen
+		err = server.ListenAndServe()
 	}
 
-	err = http.ListenAndServe(listen, api.Router(context))
-	if err != nil {
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("unable to serve: %s", err)
 	}
 
-	return err
+	return nil
 }
 
 func makeCmdAPIServe() *commander.Command {

cmd/cmd.go

@@ -97,7 +97,7 @@ package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
 			makeCmdConfig(),
-			makeCmdDb(),
+			makeCmdDB(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),
@@ -118,7 +118,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
 	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {

cmd/config_show.go

@@ -5,19 +5,30 @@ import (
 	"fmt"
 
 	"github.com/smira/commander"
+	yaml "gopkg.in/yaml.v3"
 )
 
-func aptlyConfigShow(cmd *commander.Command, args []string) error {
+func aptlyConfigShow(_ *commander.Command, _ []string) error {
+	showYaml := context.Flags().Lookup("yaml").Value.Get().(bool)
 
 	config := context.Config()
-	prettyJSON, err := json.MarshalIndent(config, "", "    ")
 
-	if err != nil {
-		return fmt.Errorf("unable to dump the config file: %s", err)
+	if showYaml {
+		yamlData, err := yaml.Marshal(&config)
+		if err != nil {
+			return fmt.Errorf("error marshaling to YAML: %s", err)
+		}
+
+		fmt.Println(string(yamlData))
+	} else {
+		prettyJSON, err := json.MarshalIndent(config, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to dump the config file: %s", err)
+		}
+
+		fmt.Println(string(prettyJSON))
 	}
 
-	fmt.Println(string(prettyJSON))
-
 	return nil
 }
 
@@ -35,5 +46,6 @@ Example:
 
 `,
 	}
+	cmd.Flag.Bool("yaml", false, "show yaml config")
 	return cmd
 }

cmd/db.go

@@ -4,13 +4,13 @@ import (
 	"github.com/smira/commander"
 )
 
-func makeCmdDb() *commander.Command {
+func makeCmdDB() *commander.Command {
 	return &commander.Command{
 		UsageLine: "db",
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
-			makeCmdDbCleanup(),
-			makeCmdDbRecover(),
+			makeCmdDBCleanup(),
+			makeCmdDBRecover(),
 		},
 	}
 }

cmd/db_cleanup.go

@@ -12,7 +12,7 @@ import (
 )
 
 // aptly db cleanup
-func aptlyDbCleanup(cmd *commander.Command, args []string) error {
+func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -26,6 +26,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
+	referencedAppStreamFiles := []string{}
 
 	// used only in verbose mode to report package use source
 	packageRefSources := map[string][]string{}
@@ -48,13 +49,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 			if verbose {
 				description := fmt.Sprintf("mirror %s", repo.Name)
-				repo.RefList().ForEach(func(key []byte) error {
+				_ = repo.RefList().ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
 			}
 		}
 
+		for _, poolPath := range repo.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -81,7 +86,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 			if verbose {
 				description := fmt.Sprintf("local repo %s", repo.Name)
-				repo.RefList().ForEach(func(key []byte) error {
+				_ = repo.RefList().ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -113,11 +118,16 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 		if verbose {
 			description := fmt.Sprintf("snapshot %s", snapshot.Name)
-			snapshot.RefList().ForEach(func(key []byte) error {
+			_ = snapshot.RefList().ForEach(func(key []byte) error {
 				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 				return nil
 			})
 		}
+
+		for _, poolPath := range snapshot.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -146,7 +156,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			if verbose {
 				description := fmt.Sprintf("published repository %s:%s/%s component %s",
 					published.Storage, published.Prefix, published.Distribution, component)
-				published.RefList(component).ForEach(func(key []byte) error {
+				_ = published.RefList(component).ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -236,6 +246,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
 	sort.Strings(referencedFiles)
 	context.Progress().ShutdownBar()
 
@@ -291,9 +302,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func makeCmdDbCleanup() *commander.Command {
+func makeCmdDBCleanup() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbCleanup,
+		Run:       aptlyDBCleanup,
 		UsageLine: "cleanup",
 		Short:     "cleanup DB and package pool",
 		Long: `

cmd/db_recover.go

@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 
 	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
-func aptlyDbRecover(cmd *commander.Command, args []string) error {
+func aptlyDBRecover(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -16,14 +19,19 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = goleveldb.RecoverDB(context.DBPath())
+	if err = goleveldb.RecoverDB(context.DBPath()); err != nil {
+		return err
+	}
+
+	context.Progress().Printf("Checking database integrity...\n")
+	err = checkIntegrity()
 
 	return err
 }
 
-func makeCmdDbRecover() *commander.Command {
+func makeCmdDBRecover() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbRecover,
+		Run:       aptlyDBRecover,
 		UsageLine: "recover",
 		Short:     "recover DB after crash",
 		Long: `
@@ -38,3 +46,36 @@ Example:
 
 	return cmd
 }
+
+func checkIntegrity() error {
+	return context.NewCollectionFactory().LocalRepoCollection().ForEach(checkRepo)
+}
+
+func checkRepo(repo *deb.LocalRepo) error {
+	collectionFactory := context.NewCollectionFactory()
+	repos := collectionFactory.LocalRepoCollection()
+
+	err := repos.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("load complete repo %q: %s", repo.Name, err)
+	}
+
+	dangling, err := deb.FindDanglingReferences(repo.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		return fmt.Errorf("find dangling references: %w", err)
+	}
+
+	if len(dangling.Refs) > 0 {
+		for _, ref := range dangling.Refs {
+			context.Progress().Printf("Removing dangling database reference %q\n", ref)
+		}
+
+		repo.UpdateRefList(repo.RefList().Subtract(dangling))
+
+		if err = repos.Update(repo); err != nil {
+			return fmt.Errorf("update repo: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/graph.go

@@ -4,13 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
@@ -36,12 +34,12 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 
 	buf := bytes.NewBufferString(graph.String())
 
-	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	tempfile, err := os.CreateTemp("", "aptly-graph")
 	if err != nil {
 		return err
 	}
-	tempfile.Close()
-	os.Remove(tempfile.Name())
+	_ = tempfile.Close()
+	_ = os.Remove(tempfile.Name())
 
 	format := context.Flags().Lookup("format").Value.String()
 	output := context.Flags().Lookup("output").Value.String()
@@ -80,10 +78,6 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	defer func() {
-		_ = os.Remove(tempfilename)
-	}()
-
 	if output != "" {
 		err = utils.CopyFile(tempfilename, output)
 		if err != nil {
@@ -91,23 +85,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("Output saved to %s\n", output)
+		_ = os.Remove(tempfilename)
 	} else {
 		command := getOpenCommand()
-		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+		fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename)
 
 		args := strings.Split(command, " ")
 
 		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
 		viewer.Stderr = os.Stderr
-		if err = viewer.Start(); err == nil {
-			// Wait for a second so that the visualizer has a chance to
-			// open the input file. This needs to be done even if we're
-			// waiting for the visualizer as it can be just a wrapper that
-			// spawns a browser tab and returns right away.
-			defer func(t <-chan time.Time) {
-				<-t
-			}(time.After(time.Second))
-		}
+		err = viewer.Start()
 	}
 
 	return err

cmd/mirror.go

@@ -9,18 +9,18 @@ import (
 )
 
 func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
-	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
-		return nil, nil
-	}
-
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(!ignoreSignatures) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -20,6 +20,11 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -37,12 +42,12 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs, downloadInstaller)
+		downloadSources, downloadUdebs, downloadInstaller, downloadAppStream)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -59,7 +64,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
@@ -96,10 +101,11 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")

cmd/mirror_edit.go

@@ -28,12 +28,15 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 	}
 
 	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter = flag.Value.String()
+			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-appstream":
+			repo.DownloadAppStream = flag.Value.Get().(bool)
 		case "with-installer":
 			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
@@ -43,6 +46,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		case "archive-url":
 			repo.SetArchiveRoot(flag.Value.String())
 			fetchMirror = true
+		case "ignore-signatures":
+			ignoreSignatures = true
 		}
 	})
 
@@ -50,6 +55,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
 	}
 
+	if repo.IsFlat() && repo.DownloadAppStream {
+		return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -69,7 +78,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 		}
 
-		err = repo.Fetch(context.Downloader(), verifier)
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
@@ -101,9 +110,10 @@ Example:
 	}
 
 	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_list.go

@@ -24,7 +24,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return aptlyMirrorListTxt(cmd, args)
 }
 
-func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -32,7 +32,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
 
 	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -42,7 +42,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -65,18 +65,18 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		repos[i] = repo
 		i++
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/mirror_show.go

@@ -27,7 +27,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	return aptlyMirrorShowTxt(cmd, args)
 }
 
-func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
 		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	downloadAppStream := No
+	if repo.DownloadAppStream {
+		downloadAppStream = Yes
+	}
+	fmt.Printf("Download AppStream: %s\n", downloadAppStream)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := No
@@ -86,14 +91,14 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList(), collectionFactory)
+			_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
 	return err
 }
 
-func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -119,7 +124,7 @@ func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				repo.Packages = append(repo.Packages, p.GetFullName())
 				return nil
 			})

cmd/mirror_update.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"sync"
 
@@ -41,24 +42,37 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.DownloadAppStream && !repo.IsFlat() {
+		context.Progress().Printf("Downloading AppStream metadata...\n")
+		err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
+			context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	if repo.Filter != "" {
 		context.Progress().Printf("Applying filter...\n")
 		var filterQuery deb.PackageQuery
@@ -82,10 +96,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	)
 
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+	latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
 	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -96,7 +111,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			collectionFactory.RemoteRepoCollection().Update(repo)
+			_ = collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
@@ -161,7 +176,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 					var e error
 
 					// provision download location
-					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+						task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+					} else {
+						var file *os.File
+						file, e = os.CreateTemp("", task.File.Filename)
+						if e == nil {
+							task.TempDownPath = file.Name()
+							_ = file.Close()
+						}
+					}
 					if e != nil {
 						pushError(e)
 						continue
@@ -173,7 +197,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
-						ignoreMismatch)
+						ignoreChecksums)
 					if e != nil {
 						pushError(e)
 						continue
@@ -197,6 +221,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	defer func() {
+		for _, task := range queue {
+			if task.TempDownPath == "" {
+				continue
+			}
+
+			if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+				fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+			}
+		}
+	}()
+
 	// Import downloaded files
 	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 
@@ -235,13 +271,13 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
-	repo.FinalizeDownload(collectionFactory, context.Progress())
+	_ = repo.FinalizeDownload(collectionFactory, context.Progress())
 	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
 	return err
 }
 
@@ -266,6 +302,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
+	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

cmd/package_search.go

@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	if len(args) == 1 {
-		q, err = query.Parse(args[0])
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -36,7 +40,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }
@@ -49,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
 		Long: `
 Command search displays list of packages in whole DB that match package query.
 
+Use '@file' to read query from file or '@-' for stdin.
 If query is not specified, all the packages are displayed.
 
 Example:

cmd/package_show.go

@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -80,8 +84,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false, false)
-		w.Flush()
+		_ = p.Stanza().WriteTo(w, p.IsSource, false, false)
+		_ = w.Flush()
 		fmt.Printf("\n")
 
 		if withFiles {
@@ -105,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p, collectionFactory)
+			_ = printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 
@@ -130,6 +134,8 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
     $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'

cmd/publish.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+
+	var gpgKeys []string
+
+	// CLI args have priority over config
+	cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
+	if len(cliKeys) > 0 {
+		gpgKeys = cliKeys
+	} else if len(context.Config().GpgKeys) > 0 {
+		gpgKeys = context.Config().GpgKeys
+	}
+
+	for _, gpgKey := range gpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 
 }
 
+type gpgKeyFlag struct {
+	gpgKeys []string
+}
+
+func (k *gpgKeyFlag) Set(value string) error {
+	k.gpgKeys = append(k.gpgKeys, value)
+	return nil
+}
+
+func (k *gpgKeyFlag) Get() interface{} {
+	return k.gpgKeys
+}
+
+func (k *gpgKeyFlag) String() string {
+	return strings.Join(k.gpgKeys, ",")
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",
@@ -34,10 +66,26 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishDrop(),
 			makeCmdPublishList(),
 			makeCmdPublishRepo(),
+			makeCmdPublishShow(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishSource(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
-			makeCmdPublishShow(),
+		},
+	}
+}
+
+func makeCmdPublishSource() *commander.Command {
+	return &commander.Command{
+		UsageLine: "source",
+		Short:     "manage sources of published repository",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSourceAdd(),
+			makeCmdPublishSourceDrop(),
+			makeCmdPublishSourceList(),
+			makeCmdPublishSourceRemove(),
+			makeCmdPublishSourceReplace(),
+			makeCmdPublishSourceUpdate(),
 		},
 	}
 }

cmd/publish_list.go

@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return aptlyPublishListTxt(cmd, args)
 }
 
-func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -34,7 +34,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
 	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
+		e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
 		if e != nil {
 			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
 				repo.StoragePrefix(), repo.Distribution, repo.Components())
@@ -53,7 +53,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(published)
 
@@ -77,7 +77,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
@@ -99,7 +99,7 @@ func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].GetPath() < repos[j].GetPath()

cmd/publish_repo.go

@@ -34,7 +34,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -48,8 +48,12 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+	cmd.Flag.String("version", "", "version of the release")
 
 	return cmd
 }

cmd/publish_show.go

@@ -24,7 +24,7 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	return aptlyPublishShowTxt(cmd, args)
 }
 
-func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]
@@ -52,7 +52,8 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
 
 	fmt.Printf("Sources:\n")
-	for component, sourceID := range repo.Sources {
+	for _, component := range repo.Components() {
+		sourceID := repo.Sources[component]
 		var name string
 		if repo.SourceKind == deb.SourceSnapshot {
 			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
@@ -76,7 +77,7 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	distribution := args[0]

cmd/publish_snapshot.go

@@ -116,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	origin := context.Flags().Lookup("origin").Value.String()
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -132,6 +133,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 	published.Label = context.Flags().Lookup("label").Value.String()
 	published.Suite = context.Flags().Lookup("suite").Value.String()
+	published.Codename = context.Flags().Lookup("codename").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
 
@@ -148,9 +150,21 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
 	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+		_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -161,11 +175,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
-			"the same package pool.\n")
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -225,7 +238,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -239,8 +252,12 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
+	cmd.Flag.String("version", "", "version of the release")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_source_add.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if exists {
+			return fmt.Errorf("unable to add: component '%s' has already been added", component)
+		}
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceAdd,
+		UsageLine: "add <distribution> <source>",
+		Short:     "add source components to a published repo",
+		Long: `
+The command adds components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source add -component=contrib wheezy ppa wheezy-contrib
+
+This command assigns the snapshot wheezy-contrib to the component contrib and
+adds it to published repository revision of ppa/wheezy.
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_drop.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published.DropRevision()
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("Source changes have been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishSourceDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceDrop,
+		UsageLine: "drop <distribution>",
+		Short:     "drop pending source component changes of a published repository",
+		Long: `
+Remove all pending changes what would be applied with a subsequent 'aptly publish update'.
+
+Example:
+
+    $ aptly publish source drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_list.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	if published.Revision == nil {
+		return fmt.Errorf("unable to list: no source changes exist")
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishSourceListJSON(published)
+	}
+
+	return aptlyPublishSourceListTxt(published)
+}
+
+func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	fmt.Printf("Sources:\n")
+	for _, component := range revision.Components() {
+		name := revision.Sources[component]
+		fmt.Printf("  %s: %s [%s]\n", component, name, published.SourceKind)
+	}
+
+	return nil
+}
+
+func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	output, err := json.MarshalIndent(revision.SourceList(), "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+func makeCmdPublishSourceList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceList,
+		UsageLine: "list <distribution>",
+		Short:     "lists revision of published repository",
+		Long: `
+Command lists sources of a published repository.
+
+Example:
+
+    $ aptly publish source list wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
+	}
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_remove.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(components) == 0 {
+		return fmt.Errorf("unable to remove: missing components, specify at least one component")
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for _, component := range components {
+		name, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to remove: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		delete(sources, component)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceRemove,
+		UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
+		Short:     "remove source components from a published repo",
+		Long: `
+The command removes source components (snapshot / local repo) from a published repository.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be removed, e.g.:
+
+Example:
+
+	$ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_replace.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+	context.Progress().Printf("Replacing source list...\n")
+	clear(sources)
+
+	for i, component := range components {
+		name := names[i]
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceReplace() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceReplace,
+		UsageLine: "replace <distribution> <source>",
+		Short:     "replace the source components of a published repository",
+		Long: `
+The command replaces the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_update.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceUpdate,
+		UsageLine: "update <distribution> <source>",
+		Short:     "update the source components of a published repository",
+		Long: `
+The command updates the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source update -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -11,7 +11,10 @@ import (
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
-	var err error
+	var (
+		err   error
+		names []string
+	)
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
@@ -23,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	distribution := args[0]
 	param := "."
 
-	var (
-		names    []string
-		snapshot *deb.Snapshot
-	)
-
 	if len(args) == len(components)+2 {
 		param = args[1]
 		names = args[2:]
@@ -42,16 +40,16 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	if published.SourceKind != deb.SourceSnapshot {
-		return fmt.Errorf("unable to update: not a snapshot publish")
+		return fmt.Errorf("unable to switch: not a published snapshot repository")
 	}
 
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	publishedComponents := published.Components()
@@ -63,17 +61,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
 	}
 
+	snapshotCollection := collectionFactory.SnapshotCollection()
 	for i, component := range components {
 		if !utils.StrSliceHasItem(publishedComponents, component) {
-			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+			return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
 		}
 
-		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
+		snapshot, err := snapshotCollection.ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -100,7 +99,19 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -112,14 +123,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
 		if err != nil {
-			return fmt.Errorf("unable to update: %s", err)
+			return fmt.Errorf("unable to switch: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -127,15 +137,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
-		Short:     "update published repository by switching to new snapshot",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
+		Short:     "update published repository by switching to new source",
 		Long: `
-Command switches in-place published snapshots with new snapshot contents. All
+Command switches in-place published snapshots with new source contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
 For multiple component repositories, flag -component should be given with
-list of components to update. Corresponding snapshots should be given in the
+list of components to update. Corresponding sources should be given in the
 same order, e.g.:
 
 	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
@@ -149,7 +159,7 @@ This command would switch published repository (with one component) named ppa/wh
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -160,7 +170,10 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
+	cmd.Flag.String("version", "", "version of the release")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -31,18 +31,14 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != deb.SourceLocalRepo {
-		return fmt.Errorf("unable to update: not a local repository publish")
-	}
-
 	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	components := published.Components()
-	for _, component := range components {
-		published.UpdateLocalRepo(component)
+	result, err := published.Update(collectionFactory, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -64,7 +60,27 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
+	if context.Flags().IsSet("origin") {
+		published.Origin = context.Flags().Lookup("origin").Value.String()
+	}
+
+	if context.Flags().IsSet("label") {
+		published.Label = context.Flags().Lookup("label").Value.String()
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -76,14 +92,15 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
 	}
 
-	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -92,15 +109,21 @@ func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
 		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
-		Short:     "update published local repository",
+		Short:     "update published repository",
 		Long: `
-Command re-publishes (updates) published local repository. <distribution>
-and <prefix> should be occupied with local repository published
-using command aptly publish repo. Update happens in-place with
-minimum possible downtime for published repository.
+The command updates updates a published repository after applying pending changes to the sources.
 
-For multiple component published repositories, all local repositories
-are updated.
+For published local repositories:
+
+    * update to match local repository contents
+
+For published snapshots:
+
+    * switch components to new snapshot
+
+The update happens in-place with minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories are updated.
 
 Example:
 
@@ -108,7 +131,7 @@ Example:
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -118,7 +141,12 @@ Example:
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+    cmd.Flag.String("origin", "", "overwrite origin name to publish")
+    cmd.Flag.String("label", "", "overwrite label to publish")
+    cmd.Flag.String("version", "", "version of the release")
 
 	return cmd
 }

cmd/repo_add.go

@@ -91,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 func makeCmdRepoAdd() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoAdd,
-		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		UsageLine: "add <name> (<package file.deb>|<directory>)...",
 		Short:     "add packages to local repository",
 		Long: `
 Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.

cmd/repo_include.go

@@ -29,7 +29,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
-	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
 	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
 	collectionFactory := context.NewCollectionFactory()
@@ -83,14 +86,14 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 func makeCmdRepoInclude() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoInclude,
-		UsageLine: "include <file.changes>|<directory> ...",
+		UsageLine: "include (<file.changes>|<directory>)...",
 		Short:     "add packages to local repositories based on .changes files",
 		Long: `
 Command include looks for .changes files in list of arguments or specified directories. Each
 .changes file is verified, parsed, referenced files are put into separate temporary directory
 and added into local repository. Successfully imported files are removed by default.
 
-Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+Additionally uploads could be restricted with 'uploaders.json' file. Rules in this file control
 uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
 
 Example:

cmd/repo_list.go

@@ -24,7 +24,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return aptlyRepoListTxt(cmd, args)
 }
 
-func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -32,7 +32,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -47,7 +47,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -71,12 +71,12 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 	var err error
 
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
@@ -87,7 +87,7 @@ func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/repo_move.go

@@ -110,13 +110,24 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 	}
 
-	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+	toProcess, err := srcList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            dstList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -179,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -38,19 +38,23 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
 	}
 
 	list.PrepareIndex()
-	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	toRemove, err := list.Filter(deb.FilterOptions{Queries: queries})
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *deb.Package) error {
+	_ = toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
 		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil
@@ -81,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/repo_show.go

@@ -25,7 +25,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 	return aptlyRepoShowTxt(cmd, args)
 }
 
-func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -52,13 +52,13 @@ func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList(), collectionFactory)
+		_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 	}
 
 	return err
 }
 
-func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]

cmd/serve.go

@@ -29,7 +29,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}

cmd/snapshot_create.go

@@ -84,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotCreate,
-		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		UsageLine: "create <name> (from mirror <mirror-name> | from repo <repo-name> | empty)",
 		Short:     "creates snapshot of mirror (local repository) contents",
 		Long: `
 Command create <name> from mirror makes persistent immutable snapshot of remote

cmd/snapshot_filter.go

@@ -60,14 +60,25 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i, arg := range args[2:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := packageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            nil,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
@@ -96,6 +107,8 @@ Command filter does filtering in snapshot <source>, producing another
 snapshot <destination>. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'

cmd/snapshot_list.go

@@ -23,7 +23,7 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	return aptlySnapshotListTxt(cmd, args)
 }
 
-func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
 	var err error
 
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -33,7 +33,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
 	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
-		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
 			return nil
 		})
@@ -59,7 +59,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
 	var err error
 
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
@@ -68,7 +68,7 @@ func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
 
 	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
 	i := 0
-	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+	_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 		jsonSnapshots[i] = snapshot
 		i++
 		return nil

cmd/snapshot_pull.go

@@ -88,7 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-3)
 	for i, arg := range args[3:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -97,7 +101,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := sourcePackageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  !noDeps,
+		Source:            packageList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -105,14 +116,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	alreadySeen := map[string]bool{}
 
-	result.ForEachIndexed(func(pkg *deb.Package) error {
+	_ = result.ForEachIndexed(func(pkg *deb.Package) error {
 		key := pkg.Architecture + "_" + pkg.Name
 		_, seen := alreadySeen[key]
 
 		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
 		if !noRemove && !seen {
 			// Remove all packages with the same name and architecture
-			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
 			for _, p := range pS {
 				packageList.Remove(p)
 				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
@@ -121,7 +132,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 		// If !allMatches, add only first matching name-arch package
 		if !seen || allMatches {
-			packageList.Add(pkg)
+			_ = packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
 		}
 
@@ -160,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server

cmd/snapshot_search.go

@@ -78,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	list.PrepareIndex()
 
 	if len(args) == 2 {
-		q, err = query.Parse(args[1])
+		value, err := GetStringOrFileContent(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -103,8 +107,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 	}
 
-	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := list.Filter(deb.FilterOptions{
+		Queries:           []deb.PackageQuery{q},
+		WithDependencies:  withDeps,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -114,7 +123,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }
@@ -129,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
 
 If query is not specified, all the packages are displayed.
 
+Use '@file' syntax to read package query from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'

cmd/snapshot_show.go

@@ -25,7 +25,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 	return aptlySnapshotShowTxt(cmd, args)
 }
 
-func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 	var err error
 	name := args[0]
 	collectionFactory := context.NewCollectionFactory()
@@ -79,13 +79,13 @@ func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+		_ = ListPackagesRefList(snapshot.RefList(), collectionFactory)
 	}
 
 	return err
 }
 
-func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 	var err error
 
 	name := args[0]
@@ -139,7 +139,7 @@ func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
 				return nil
 			})

cmd/string_or_file_flag.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"strings"
+
+	"github.com/smira/flag"
+)
+
+// StringOrFileFlag is a custom flag type that can handle both string input and file input.
+// If the input starts with '@', it is treated as a filename and the contents are read from the file.
+// If the input is '@-', the contents are read from stdin.
+type StringOrFileFlag struct {
+	value string
+}
+
+func (s *StringOrFileFlag) String() string {
+	return s.value
+}
+
+func (s *StringOrFileFlag) Set(value string) error {
+	var err error
+	s.value, err = GetStringOrFileContent(value)
+	return err
+}
+
+func (s *StringOrFileFlag) Get() any {
+	return s.value
+}
+
+func AddStringOrFileFlag(flagSet *flag.FlagSet, name string, value string, usage string) *StringOrFileFlag {
+	result := &StringOrFileFlag{value: value}
+	flagSet.Var(result, name, usage)
+	return result
+}
+
+func GetStringOrFileContent(value string) (string, error) {
+	if !strings.HasPrefix(value, "@") {
+		return value, nil
+	}
+
+	filename := strings.TrimPrefix(value, "@")
+	var data []byte
+	var err error
+	if filename == "-" { // Read from stdin
+		data, err = io.ReadAll(os.Stdin)
+	} else {
+		data, err = os.ReadFile(filename)
+	}
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}

cmd/task_run.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mattn/go-shellwords"
+	shellwords "github.com/mattn/go-shellwords"
 	"github.com/smira/commander"
 )
 
@@ -31,7 +31,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		defer file.Close()
+		defer func() { _ = file.Close() }()
 
 		scanner := bufio.NewScanner(file)
 
@@ -62,11 +62,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 			text, _ = reader.ReadString('\n')
 			if text == "\n" {
 				break
-			} else {
-				text = strings.TrimSpace(text) + ","
-				parsedArgs, _ := shellwords.Parse(text)
-				cmdArgs = append(cmdArgs, parsedArgs...)
 			}
+			text = strings.TrimSpace(text) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
 		}
 
 		if len(cmdArgs) == 0 {
@@ -132,7 +131,7 @@ func formatCommands(args []string) [][]string {
 func makeCmdTaskRun() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyTaskRun,
-		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		UsageLine: "run (-filename=<filename> | <commands>...)",
 		Short:     "run aptly tasks",
 		Long: `
 Command helps organise multiple aptly commands in one single aptly task, running as single thread.

codecov.yml

@@ -3,5 +3,5 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 0%
+        threshold: 2%
         if_ci_failed: error

completion.d/_aptly

@@ -185,6 +185,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
                         ;;
                     list)
@@ -211,6 +212,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-max-tries=[max download tries till process fails with download error]:number: " \
                             "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
+                            "-latest=[download only latest version of each package (per architecture)]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     rename)
@@ -223,6 +225,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     search)
@@ -457,6 +460,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-distribution=[distribution name to publish]:distribution:($dists)"
                             "-label=[label to publish]:label: "
                             "-suite=[suite to publish]:suite: "
+                            "-codename=[codename to publish]:codename: "
                             "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
                             "-origin=[origin name to publish]:origin: "
                             ${components_options[@]}