Merge pull request #1406 from aptly-dev/release/1.6.0

Release/1.6.0

.dockerignore

@@ -0,0 +1,10 @@
+.go/
+.git/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+unit.out
+aptly.test
+build/
+dpkgs/

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 240
+ignore = E126,E241,E741,W504
+include =
+    system
+exclude =
+    system/env

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: aptly-dev

.github/workflows/ci.yml

@@ -0,0 +1,285 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  test:
+    name: "Test (Ubuntu 22.04)"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: "Install packages"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
+
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Run flake8"
+        run: |
+          make flake8
+
+      - name: "Read go version from go.mod"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Install Azurite"
+        id: azuright
+        uses: potatoqualitee/azuright@v1.1
+        with:
+          directory: ${{ runner.temp }}
+
+      - name: "Run Unit Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make test
+
+      - name: "Run Benchmark"
+        run: |
+          COVERAGE_DIR=${{ runner.temp }} make bench
+
+      - name: "Run System Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make system-test
+
+      - name: "Merge code coverage"
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+
+      - name: "Upload code coverage"
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.txt
+
+  ci-debian-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        arch: ["amd64", "i386" , "arm64" , "armhf"]
+        include:
+          - name: "Debian 13/testing"
+            suite: trixie
+            image: debian:trixie-slim
+          - name: "Debian 12/bookworm"
+            suite: bookworm
+            image: debian:bookworm-slim
+          - name: "Debian 11/bullseye"
+            suite: bullseye
+            image: debian:bullseye-slim
+          - name: "Debian 10/buster"
+            suite: buster
+            image: debian:buster-slim
+          - name: "Ubuntu 24.04"
+            suite: noble
+            image: ubuntu:24.04
+          - name: "Ubuntu 22.04"
+            suite: jammy
+            image: ubuntu:22.04
+          - name: "Ubuntu 20.04"
+            suite: focal
+            image: ubuntu:20.04
+    container:
+      image: ${{ matrix.image }}
+      env:
+        APT_LISTCHANGES_FRONTEND: none
+        DEBIAN_FRONTEND: noninteractive
+    steps:
+      - name: "Install packages"
+        run: |
+          apt-get update
+          apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
+                          binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf \
+                          libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
+                          gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read go version from go.mod"
+        run: |
+          gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
+          echo "Go Version: $gover"
+          echo "GOVER=$gover" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Build Debian packages"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make dpkg DEBARCH=${{ matrix.arch }}
+
+      - name: "Check aptly credentials"
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          found=no
+          if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
+            found=yes
+          fi
+          echo "Aptly credentials available: $found"
+          echo "FOUND=$found" >> $GITHUB_OUTPUT
+        id: aptlycreds
+
+      - name: "Publish CI release to aptly"
+        if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
+
+      - name: "Publish release to aptly"
+        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
+
+  ci-binary-build:
+    name: "Build"
+    needs: test
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, freebsd, darwin]
+        goarch: ["386", "amd64", "arm", "arm64"]
+        exclude:
+          - goos: darwin
+            goarch: 386
+          - goos: darwin
+            goarch: arm
+    steps:
+      - name: "Checkout repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: "Ensure CI build"
+        if: github.ref == 'refs/heads/master'
+        run: |
+          echo "FORCE_CI=true" >> $GITHUB_OUTPUT
+        id: force_ci
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
+        env:
+          GOBIN: /usr/local/bin
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
+
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
+        if: startsWith(github.event.ref, 'refs/tags')
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+
+  gh-release:
+    name: "Github Release"
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: ci-binary-build
+    if: startsWith(github.event.ref, 'refs/tags')
+    steps:
+      - name: "Download Artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: out/
+
+      - name: "Release"
+        uses: softprops/action-gh-release@v2
+        with:
+          files: "out/**/aptly_*.zip"

.github/workflows/golangci-lint.yml

@@ -0,0 +1,72 @@
+name: golangci-lint
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+  # Optional: allow read access to pull request. Use with `only-new-issues` option.
+  # pull-requests: read
+
+jobs:
+  golangci:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: "Read go version from go.mod"
+        run: |
+          echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
+        id: goversion
+
+      - name: "Setup Go"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ steps.goversion.outputs.GOVER }}
+
+      - name: Create VERSION file
+        run: |
+          make -s version | tr -d '\n' > VERSION
+        shell: sh
+
+      - name: Install and initialize swagger
+        run: |
+          go install github.com/swaggo/swag/cmd/swag@latest
+          swag init -q --markdownFiles docs
+        shell: sh
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v4
+        with:
+          # Require: The version of golangci-lint to use.
+          # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
+          # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
+          version: v1.54.1
+
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+
+          # Optional: golangci-lint command line arguments.
+          #
+          # Note: By default, the `.golangci.yml` file should be at the root of the repository.
+          # The location of the configuration file can be changed by using `--config=`
+          # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
+
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true
+
+          # Optional: if set to true, then all caching functionality will be completely disabled,
+          #           takes precedence over all other caching options.
+          # skip-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/go/pkg.
+          # skip-pkg-cache: true
+
+          # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
+          # skip-build-cache: true
+
+          # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
+          # install-mode: "goinstall"

.github/workflows/scripts/upload-artifacts.sh

@@ -0,0 +1,167 @@
+#!/bin/sh
+
+set -e
+
+builds=build/
+packages=${builds}*.deb
+folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
+aptly_user="$APTLY_USER"
+aptly_password="$APTLY_PASSWORD"
+aptly_api="https://aptly-ops.aptly.info"
+version=`make version`
+
+action=$1
+dist=$2
+
+usage() {
+    echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
+    echo "       $0 release" >&2
+}
+
+# repos and publish must be created beforehand:
+#!/bin/sh
+#for dist in buster bullseye bookworm focal jammy noble
+#do
+#        for build in ci release
+#        do
+#            echo
+#            echo "# Creating and publishing $build/$dist"
+#            aptly repo create -distribution=$dist -component=main aptly-$build-$dist
+#            aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
+#                               -distribution=$dist -batch -keyring=aptly.pub \
+#                               aptly-$build-$dist \
+#                               s3:repo.aptly.info:$build
+#       done
+#done
+
+if [ -z "$action" ]; then
+    usage
+    exit 1
+fi
+
+if [ "action" = "ci" ] && [ -z "$dist" ]; then
+    usage
+    exit 1
+fi
+
+if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
+    usage
+    echo Error: please set APTLY_USER and APTLY_PASSWORD
+    exit 1
+fi
+
+echo "Publishing version '$version' to $action for $dist...\n"
+
+upload()
+{
+    echo "\nUploading files:"
+    for file in $packages; do
+        echo " - $file"
+        jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+    done
+}
+
+cleanup() {
+    echo "\nCleanup..."
+    jsonret=`curl -fsS -X DELETE  -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
+}
+trap cleanup EXIT
+
+wait_task()
+{
+    _id=$1
+    _success=0
+    for t in `seq 180`
+    do
+        jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
+        _state=`echo $jsonret | jq .State`
+        if [ "$_state" = "2" ]; then
+            _success=1
+            curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id
+            break
+        fi
+        if [ "$_state" = "3" ]; then
+            echo Error: task failed
+            return 1
+        fi
+        sleep 1
+    done
+    if [ "$_success" -ne 1 ]; then
+        echo Error: task timeout
+        return 1
+    fi
+    return 0
+}
+
+add_packages() {
+    _aptly_repository=$1
+    _folder=$2
+    jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$_aptly_repository/file/$_folder?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: adding packages to $_aptly_repository failed"
+        exit 1
+    fi
+}
+
+update_publish() {
+    _publish=$1
+    _dist=$2
+    jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+        '{"AcquireByHash": true, "MultiDist": true,
+          "Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
+        -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
+    _task_id=`echo $jsonret | jq .ID`
+    wait_task $_task_id
+    if [ "$?" -ne 0 ]; then
+        echo "Error: publish failed"
+        exit 1
+    fi
+}
+
+if [ "$action" = "ci" ]; then
+    if echo "$version" | grep -vq "+"; then
+       # skip ci when on release tag
+       exit 0
+    fi
+
+    aptly_repository=aptly-ci-$dist
+    aptly_published=s3:repo.aptly.info:ci
+
+elif [ "$action" = "release" ]; then
+    aptly_repository=aptly-release-$dist
+    aptly_published=s3:repo.aptly.info:release
+fi
+
+upload
+
+echo "\nAdding packages to $aptly_repository ..."
+add_packages $aptly_repository $folder
+
+echo "\nUpdating published repo $aptly_published ..."
+update_publish $aptly_published $dist
+
+# if [ "$action" = "OBSOLETErelease" ]; then
+#     aptly_repository=aptly-release
+#     aptly_snapshot=aptly-$version
+#     aptly_published=s3:repo.aptly.info:./squeeze
+#
+#     echo "\nAdding packages to $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
+#     echo
+#
+#     echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
+#     curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
+#         "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
+#     echo
+#
+#     echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
+#     curl -fsS -X PUT -H 'Content-Type: application/json' --data \
+#         "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
+#                                    \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
+#                                                  \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
+#         -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
+#     echo
+# fi
+

.gitignore

@@ -2,11 +2,14 @@
 *.o
 *.a
 *.so
+unit.out
 
 # Folders
 _obj
 _test
 
+tmp/
+
 # Architecture specific extensions/prefixes
 *.[568vq]
 [568vq].out
@@ -22,8 +25,7 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
-coverage*.out
+coverage.txt
 
 *.pyc
 
@@ -33,6 +35,42 @@ root/
 man/aptly.1.html
 man/aptly.1.ronn
 
-.goxc.local.json
-
 system/env/
+
+# created by make build for release artifacts
+VERSION
+aptly.test
+
+build/
+
+system/files/aptly2.gpg~
+system/files/aptly2_passphrase.gpg~
+
+*.creds
+
+.go/
+obj-x86_64-linux-gnu/
+obj-aarch64-linux-gnu/
+obj-arm-linux-gnueabihf/
+obj-i686-linux-gnu/
+
+# debian
+debian/.debhelper/
+debian/aptly.substvars
+debian/aptly/
+debian/debhelper-build-stamp
+debian/files
+debian/aptly-api/
+debian/*.debhelper
+debian/*.debhelper.log
+debian/aptly-api.substvars
+debian/aptly-dbg.substvars
+debian/aptly-dbg/
+usr/bin/aptly
+dpkgs/
+debian/changelog.dpkg-bak
+
+docs/docs.go
+docs/swagger.json
+docs/swagger.yaml
+docs/swagger.conf

.golangci.yml

@@ -0,0 +1,16 @@
+run:
+  tests: false
+
+
+linters:
+  disable-all: true
+  enable:
+   - goconst
+   - gofmt
+   - goimports
+   - govet
+   - ineffassign
+   - misspell
+   - revive
+   - staticcheck
+   - vetshadow

.goxc.json

@@ -1,46 +0,0 @@
-{
-	"AppName": "aptly",
-	"ArtifactsDest": "xc-out/",
-	"TasksExclude": [
-		"rmbin",
-		"go-test",
-		"go-vet"
-	],
-    "TasksAppend": [
-		"bintray"
-    ],
-    "TaskSettings": {
-        "debs": {
-            "metadata": {
-                "maintainer": "Andrey Smirnov",
-                "maintainer-email": "me@smira.ru",
-                "description": "Debian repository management tool"
-            },
-            "metadata-deb": {
-                "License": "MIT",
-                "Homepage": "https://www.aptly.info/",
-                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-				"Suggests": "graphviz"
-            },
-            "other-mapped-files": {
-                "/": "root/"
-            }
-        },
-        "bintray": {
-            "repository": "aptly",
-            "subject": "smira",
-            "package": "aptly",
-            "downloadspage": "bintray.md"
-        }
-    },
-	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
-	"Arch": "386 amd64",
-	"Os": "linux darwin freebsd",
-	"MainDirsExclude": "_man,vendor",
-	"BuildSettings": {
-		"LdFlagsXVars": {
-			"Version": "main.Version"
-		}
-	},
-	"ConfigVersion": "0.9"
-}

.travis.yml

@@ -1,47 +0,0 @@
-dist: trusty
-sudo: required
-
-language: go
-
-go:
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
-  - master
-
-go_import_path: github.com/smira/aptly
-
-addons:
-  apt:
-    packages:
-    - python-virtualenv
-    - graphviz
-
-env:
-  global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
-    - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
-before_install:
-  - virtualenv system/env
-  - . system/env/bin/activate
-  - pip install six packaging appdirs
-  - pip install -U pip setuptools
-  - pip install -r system/requirements.txt
-  - make version
-install:
-  - make prepare
-
-script: make travis
-
-matrix:
-    allow_failures:
-        - go: master
-
-notifications:
-  webhooks:
-    urls:
-      - "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: false     # default: false

AUTHORS

@@ -28,3 +28,43 @@ List of contributors, in chronological order:
 * Charles Hsu (https://github.com/charz)
 * Clemens Rabe (https://github.com/seeraven)
 * TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)
+* Shengjing Zhu (https://github.com/zhsj)
+* Nabil Bendafi (https://github.com/nabilbendafi)
+* Raphael Medaer (https://github.com/rmedaer)
+* Raul Benencia (https://github.com/rul)
+* Don Kuntz (https://github.com/dkuntz2)
+* Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)
+* Markus Muellner (https://github.com/mmianl)
+* Chuan Liu (https://github.com/chuan)
+* Samuel Mutel (https://github.com/smutel)
+* Russell Greene (https://github.com/russelltg)
+* Wade Simmons (https://github.com/wadey)
+* Steven Stone (https://github.com/smstone)
+* Josh Bayfield (https://github.com/jbayfield)
+* Boxjan (https://github.com/boxjan)
+* Mauro Regli (https://github.com/reglim)
+* Alexander Zubarev (https://github.com/strike)
+* Nicolas Dostert (https://github.com/acdn-ndostert)
+* Ryan Gonzalez (https://github.com/refi64)
+* Paul Cacheux (https://github.com/paulcacheux)
+* Nic Waller (https://github.com/sf-nwaller)
+* iofq (https://github.com/iofq)
+* Noa Resare (https://github.com/nresare)
+* Ramon N.Rodriguez (https://github.com/runitonmetal)
+* Golf Hu (https://github.com/hudeng-go)
+* Cookie Fei (https://github.com/wuhuang26)
+* Andrey Loukhnov (https://github.com/aol-nnov)
+* Christoph Fiehe (https://github.com/cfiehe)
+* Blake Kostner (https://github.com/btkostner)
+* Leigh London (https://github.com/leighlondon)
+* Gordian Schoenherr (https://github.com/schoenherrg)

CODE_OF_CONDUCT.md

@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
 ## Enforcement
 
 Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
+reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
 complaints will be reviewed and investigated and will result in a response that
 is deemed necessary and appropriate to the circumstances. The project team is
 obligated to maintain confidentiality with regard to the reporter of an incident.

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 :+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
 
-The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
 These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
 
 ## What should I know before I get started?
@@ -11,11 +11,11 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
 
 This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
 By participating, you are expected to uphold this code.
-Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
 
 ### List of Repositories
 
-* [smira/aptly](https://github.com/smira/aptly) - aptly source code, functional tests, man page
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
 * [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
@@ -24,15 +24,15 @@ Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info)
 
 ### Reporting Bugs
 
-1. Please search for similar bug report in [issue tracker](https://github.com/smira/aptly/issues)
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
 2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
 3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
 4. If bug is reproducible with specific package, please provide link to package file.
-5. Open issue at [GitHub](https://github.com/smira/aptly/issues)
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
 
 ### Suggesting Enhancements
 
-1. Please search [issue tracker](https://github.com/smira/aptly/issues) for similar feature requests.
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
 2. Describe why enhancement is important to you.
 3. Include any additional details or implementation details.
 
@@ -40,12 +40,12 @@ Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info)
 
 There are two kinds of documentation:
 
-* [aptly website](https://www.aptly/info)
+* [aptly website](https://www.aptly.info)
 * aptly `man` page
 
 Core content is mostly the same, but website contains more information, tutorials, examples.
 
-If you want to update `man` page, please open PR to [main aptly repo](https://github.com/smira/aptly),
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
 details in [man page](#man-page) section.
 
 If you want to update website, please follow steps below:
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your Fist Code Contribution
+### Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,61 +68,117 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
 Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
 if that is possible.
 
+### Get the Source
+
+To clone the git repo, run the following commands:
+```
+git clone git@github.com:aptly-dev/aptly.git
+cd aptly
+```
+
 ## Development Setup
 
-This section describes local setup to start contributing to aptly source.
+Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
 
-### Go & Python
+### Docker Development Setup
 
-You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+This section describes the docker setup to start contributing to aptly.
 
-If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
-initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+#### Dependencies
 
-Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
-to prepend it or to skip this test if you're security conscious.  
+Install the following on your development machine:
+- docker
+- make
+- git
 
-### Forking and Cloning
+##### Docker installation on macOS
+1. Install [Docker Desktop on Mac](https://docs.docker.com/desktop/setup/install/mac-install/) (or via [Homebrew](https://brew.sh/))
+2. Allow directory sharing
+   - Open Docker Desktop
+   - Go to `Settings → Resources → File Sharing → Virtual File Shares`
+   - Add the aptly git repository path to the shared list (eg. /home/Users/john/aptly)
 
-As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
+#### Create docker container
 
-    mkdir -p ~/go/src/github.com/smira
-    cd ~/go/src/github.com/smira
-    git clone git@github.com:smira/aptly.git
-    cd aptly
+To build the development docker image, run:
+```
+make docker-image
+```
 
-For main repo under your GitHub user and add it as another Git remote:
+#### Build aptly
 
-    git remote add <user> git@github.com:<user>/aptly.git
+To build the aptly in the development docker container, run:
+```
+make docker-build
+```
 
-That way you can continue to build project as is (you don't need to adjust import paths), but you would need
-to specify your remote name when pushing branches:
+#### Running aptly commands
 
-    git push <user> <your-branch>
+To run aptly commands in the development docker container, run:
+```
+make docker-shell
+```
 
-### Dependencies
+Example:
+```
+$ make docker-shell
+aptly@b43e8473ef81:/work/src$ aptly version
+aptly version: 1.5.0+189+g0fc90dff
+```
 
-You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+#### Running unit tests
 
-    make prepare dev system/env
+In order to run aptly unit tests, enter the following:
+```
+make docker-unit-tests
+```
 
-This is usually one-time action.
+#### Running system tests
 
-### Building
+In order to run aptly system tests, enter the following:
+```
+make docker-system-tests
+```
 
-If you want to build aptly binary from your current source tree, run:
+#### Running golangci-lint
+
+In order to run aptly unit tests, run:
+```
+make docker-lint
+```
+
+#### More info
+
+Run `make help` for more information.
+
+
+### Local Development Setup
+
+This section describes local setup to start contributing to aptly.
+
+#### Dependencies
+
+Building aptly requires go version 1.22.
+
+On Debian bookworm with backports enabled, go can be installed with:
+
+    apt install -t bookworm-backports golang-go
+
+#### Building
+
+To build aptly, run:
+
+    make build
+
+Run aptly:
+
+    build/aptly
+
+To install aptly into `$GOPATH/bin`, run:
 
     make install
 
-This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
-
-    aptly
-
-Or, if it's not on your path:
-
-    ~/go/bin/aptly
-
-### Unit-tests
+#### Unit-tests
 
 aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
 feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -131,7 +187,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
 
     make test
 
-### Functional Tests
+#### Functional Tests
 
 Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
 runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -178,27 +234,6 @@ There are some packages available under `system/files/` directory which are used
 this default location. You can run aptly under different user or by using non-default config location with non-default
 aptly root directory.
 
-### Style Checks
-
-Style checks could be run with:
-
-    make check
-
-aptly is using [gometalinter](https://github.com/alecthomas/gometalinter) to run style checks on Go code. Configuration
-for the linter could be found in [linter.json](linter.json) file. Running linters might take considerable amount of time
-unfortunately, but usually warning reported by linters hint at real code issues.
-
-Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
-
-### Vendored Code
-
-aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
-repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
-packages over packages in `$GOPATH`.
-
-If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
-Usually all you need is `dep ensure` or `dep ensure -update`.
-
 ### man Page
 
 aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
@@ -206,34 +241,13 @@ template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any comman
 final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
 verification.
 
-Man page is built with small helper [_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
 and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
 
-### Bash Completion
+### Bash and Zsh Completion
 
-Bash completion for aptly resides in the same repo under in [bash_completion.d/aptly](bash_completion.d/aptly). It's all hand-crafted.
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
 When new option or command is introduced, bash completion should be updated to reflect that change.
 
 When aptly package is being built, it automatically pulls bash completion and man page into the package.
-
-## Design
-
-This section requires future work.
-
-*TBD*
-
-### Database
-
-### Package Pool
-
-### Package
-
-### PackageList, PackageRefList
-
-### LocalRepo, RemoteRepo, Snapshot
-
-### PublishedRepository
-
-### Context
-
-### Collections, CollectionFactory

Gopkg.lock

@@ -1,191 +0,0 @@
-memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
-
-[[projects]]
-  name = "github.com/AlekSi/pointer"
-  packages = ["."]
-  revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/DisposaBoy/JsonConfigReader"
-  packages = ["."]
-  revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
-
-[[projects]]
-  name = "github.com/awalterschulze/gographviz"
-  packages = [".","ast","parser","scanner","token"]
-  revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
-  version = "v1.0"
-
-[[projects]]
-  name = "github.com/aws/aws-sdk-go"
-  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
-  revision = "2db5849d2939d93075d911138309a83235032bea"
-  version = "v1.8.0"
-
-[[projects]]
-  name = "github.com/cheggaaa/pb"
-  packages = ["."]
-  revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
-  version = "v1.0.10"
-
-[[projects]]
-  name = "github.com/gin-gonic/gin"
-  packages = [".","binding","render"]
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
-
-[[projects]]
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  revision = "1730955e3146956d6a087861380f9b4667ed5071"
-  version = "v1.26.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  revision = "553a641470496b2327abcac10b36396bd98e45c9"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/h2non/filetype"
-  packages = ["matchers"]
-  revision = "0df83c38d14ff5f653d419d480eaac286ccbc823"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/jlaffaye/ftp"
-  packages = ["."]
-  revision = "7b85eb4638a2c0473acefcfb929a98f879c15c86"
-
-[[projects]]
-  name = "github.com/jmespath/go-jmespath"
-  packages = ["."]
-  revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9"
-  version = "0.2.2"
-
-[[projects]]
-  name = "github.com/julienschmidt/httprouter"
-  packages = ["."]
-  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
-  version = "v1.1"
-
-[[projects]]
-  name = "github.com/mattn/go-runewidth"
-  packages = ["."]
-  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
-  version = "v0.0.2"
-
-[[projects]]
-  name = "github.com/mattn/go-shellwords"
-  packages = ["."]
-  revision = "005a0944d84452842197c2108bd9168ced206f78"
-  version = "v1.0.2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mkrautz/goar"
-  packages = ["."]
-  revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mxk/go-flowrate"
-  packages = ["flowrate"]
-  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/ncw/swift"
-  packages = [".","swifttest"]
-  revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/commander"
-  packages = ["."]
-  revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/flag"
-  packages = ["."]
-  revision = "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/go-aws-auth"
-  packages = ["."]
-  revision = "0070896e9d7f4f9f2d558532b2d896ce2239992a"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/go-ftp-protocol"
-  packages = ["protocol"]
-  revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/go-uuid"
-  packages = ["uuid"]
-  revision = "ed3ca8a15a931b141440a7e98e4f716eec255f7d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/go-xz"
-  packages = ["."]
-  revision = "0c531f070014e218b21f3cfca801cc992d52726d"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/smira/lzma"
-  packages = ["."]
-  revision = "7f0af6269940baa2c938fabe73e0d7ba41205683"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/syndtr/goleveldb"
-  packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
-  revision = "3c5717caf1475fd25964109a0fc640bd150fce43"
-
-[[projects]]
-  name = "github.com/ugorji/go"
-  packages = ["codec"]
-  revision = "71c2886f5a673a35f909803f38ece5810165097b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/wsxiaoys/terminal"
-  packages = ["color"]
-  revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["cast5","openpgp","openpgp/armor","openpgp/clearsign","openpgp/elgamal","openpgp/errors","openpgp/packet","openpgp/s2k","ssh/terminal"]
-  revision = "459e26527287adbc2adcc5d0d49abff9a5f315a7"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix"]
-  revision = "99f16d856c9836c42d24e7ab64ea72916925fa97"
-
-[[projects]]
-  branch = "v1"
-  name = "gopkg.in/check.v1"
-  packages = ["."]
-  revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
-
-[[projects]]
-  name = "gopkg.in/h2non/filetype.v1"
-  packages = ["types"]
-  revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
-  version = "v1.0.1"

Gopkg.toml

@@ -1,32 +0,0 @@
-
-[[dependencies]]
-  name = "github.com/gin-gonic/gin"
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
-
-[[dependencies]]
-  branch = "master"
-  name = "github.com/mkrautz/goar"
-
-[[dependencies]]
-  branch = "master"
-  name = "github.com/smira/go-uuid"
-
-[[dependencies]]
-  branch = "master"
-  name = "github.com/smira/go-xz"
-
-[[dependencies]]
-  name = "github.com/ugorji/go"
-  revision = "71c2886f5a673a35f909803f38ece5810165097b"
-
-[[dependencies]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[[dependencies]]
-  branch = "master"
-  name = "golang.org/x/sys"
-
-[[dependencies]]
-  branch = "v1"
-  name = "gopkg.in/check.v1"

Makefile

@@ -1,85 +1,219 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
-VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files gpg http query swift s3 utils
-PYTHON?=python
-TESTS?=
+GOPATH=$(shell go env GOPATH)
+VERSION=$(shell make -s version)
+PYTHON?=python3
 BINPATH?=$(GOPATH)/bin
+GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-ifeq ($(GOVERSION), devel)
-TRAVIS_TARGET=coveralls
-else
-TRAVIS_TARGET=test
-endif
+# Uncomment to update system test gold files
+# CAPTURE := "--capture"
 
-all: test check system-test
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-prepare:
-	go get -u github.com/mattn/goveralls
-	go get -u github.com/axw/gocov/gocov
-	go get -u golang.org/x/tools/cmd/cover
-	go get -u github.com/alecthomas/gometalinter
-	gometalinter --install
+prepare:  ## Install go module dependencies
+	# Prepare go modules
+	go mod verify
+	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-dev:
-	go get -u github.com/golang/dep/...
-	go get -u github.com/laher/goxc
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
+	@reltype=ci ; \
+	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
+	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
+		gittag=`git describe --tags --exact-match 2>/dev/null` ;\
+		if echo "$$gittag" | grep -q '^v[0-9]'; then \
+			reltype=release ; \
+		fi ; \
+	fi ; \
+	echo $$reltype
 
-coverage.out:
-	rm -f coverage.*.out
-	for i in $(PACKAGES); do go test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-	rm -f coverage.*.out
-
-coverage: coverage.out
-	go tool cover -html=coverage.out
-	rm -f coverage.out
-
-check: system/env
-	if [ -x travis_wait ]; then \
-		travis_wait gometalinter --config=linter.json ./...; \
+version:  ## Print aptly version
+	@ci="" ; \
+	if [ "`make -s releasetype`" = "ci" ]; then \
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
 	else \
-		gometalinter --config=linter.json ./...; \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
 	fi
-	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
+
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
+	# Generate swagger.conf
+	cp docs/swagger.conf.tpl docs/swagger.conf
+	echo "// @version $(VERSION)" >> docs/swagger.conf
+
+azurite-start:
+	azurite -l /tmp/aptly-azurite & \
+	echo $$! > ~/.azurite.pid
+
+azurite-stop:
+	@kill `cat ~/.azurite.pid`
+
+swagger: swagger-install
+	# Generate swagger docs
+	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+
+etcd-install:
+	# Install etcd
+	test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint: prepare
+	# Install golangci-lint
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
 
 install:
-	go install -v -ldflags "-X main.Version=$(VERSION)"
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	# go generate
+	@go generate
+	# go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system/env: system/requirements.txt
-	rm -rf system/env
-	virtualenv system/env
-	system/env/bin/pip install -r system/requirements.txt
+test: prepare swagger etcd-install  ## Run unit tests
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
+	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
+	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/aptly-etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
 
-system-test: install system/env
+system-test: prepare swagger etcd-install  ## Run system tests
+	# build coverage binary
+	go test -v -coverpkg="./..." -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	. system/env/bin/activate && APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
+	# Run system tests
+	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
-travis: $(TRAVIS_TARGET) check system-test
+bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
-test:
-	go test -v `go list ./... | grep -v vendor/` -gocheck.v=true
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
-coveralls: coverage.out
-	$(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
+	fi
+	# clean
+	rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
+	# Run dpkg-buildpackage
+	@buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
+	echo "$$cmd" ; \
+	$$cmd
+	lintian ../*_$(DEBARCH).changes || true
+	# cleanup
+	@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, MacOS, Linux tar)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-test:  ## Run unit tests in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		test \
+		azurite-stop
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		system-test TEST=$(TEST) \
+		azurite-stop
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc:
-	rm -rf root/
-	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
-	cp man/aptly.1 root/usr/share/man/man1
-	cp bash_completion.d/aptly root/etc/bash_completion.d
-	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
-
-man:
+man:  ## Create man pages
 	make -C man
 
-version:
-	@echo $(VERSION)
+clean:  ## remove local build and module cache
+	# Clean all generated and build files
+	test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
+	rm -rf .go/
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: coverage.out man version
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -2,17 +2,17 @@
 aptly
 =====
 
-.. image:: https://api.travis-ci.org/smira/aptly.svg?branch=master
-    :target: https://travis-ci.org/smira/aptly
+.. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/aptly-dev/aptly/actions
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.svg?branch=master
-    :target: https://coveralls.io/r/smira/aptly?branch=master
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://matrix.to/#/#aptly:gitter.im
 
-.. image:: http://goreportcard.com/badge/smira/aptly
-    :target: http://goreportcard.com/report/smira/aptly
+.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
+    :target: https://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -39,8 +39,8 @@ Current limitations:
 
 * translations are not supported yet
 
-Download
---------
+Install Stable Version
+-----------------------
 
 To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
@@ -48,7 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list
 
 And import key that is used to sign the release::
 
-    $ apt-key adv --keyserver keys.gnupg.net --recv-keys 9E3E53F19C7DE460
+    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
 
 After that you can install aptly as any other software package::
 
@@ -58,20 +58,28 @@ After that you can install aptly as any other software package::
 Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
 Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
 
-If you would like to use nightly builds (unstable), please use following repository::
+Other Binaries
+~~~~~~~~~~~~~~~~~
 
-    deb http://repo.aptly.info/ nightly main
+Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
 
-Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
+Install CI Version
+--------------------
 
-If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
+More recent versions are available as CI builds (development, might be unstable).
 
-    mkdir -p $GOPATH/src/github.com/smira/aptly
-    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
-    cd $GOPATH/src/github.com/smira/aptly
-    make install
+Debian GNU/Linux
+~~~~~~~~~~~~~~~~~
 
-Binary would be installed to ```$GOPATH/bin/aptly``.
+Install the following APT key::
+
+    sudo wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
+
+Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
+
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
+
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
 Contributing
 ------------
@@ -90,7 +98,7 @@ Vagrant:
 Docker:
 
 -    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
--    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
 
 With configuration management systems:
 
@@ -109,6 +117,10 @@ CLI for aptly API:
 -   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
 -   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
 
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/smira/aptly/cmd"
+	"github.com/aptly-dev/aptly/cmd"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

api/api.go

@@ -3,13 +3,18 @@ package api
 
 import (
 	"fmt"
+	"net/http"
 	"sort"
-	"time"
+	"strconv"
+	"strings"
+	"sync/atomic"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/rs/zerolog/log"
 )
 
 // Lock order acquisition (canonical):
@@ -18,11 +23,67 @@ import (
 //  3. SnapshotCollection
 //  4. PublishedRepoCollection
 
-// GET /api/version
+type aptlyVersion struct {
+	// Aptly Version
+	Version string `json:"Version"`
+}
+
+// @Summary Aptly Version
+// @Description **Get aptly version**
+// @Description
+// @Description **Example:**
+// @Description ```
+// @Description $ curl http://localhost:8080/api/version
+// @Description {"Version":"0.9~dev"}
+// @Description ```
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyVersion
+// @Router /api/version [get]
 func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+type aptlyStatus struct {
+	// Aptly Status
+	Status string `json:"Status" example:"'Aptly is ready', 'Aptly is unavailable', 'Aptly is healthy'"`
+}
+
+// @Summary Get Ready State
+// @Description **Get aptly ready state**
+// @Description
+// @Description Return aptly ready state:
+// @Description - `Aptly is ready` (HTTP 200)
+// @Description - `Aptly is unavailable` (HTTP 503)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus "Aptly is ready"
+// @Failure 503 {object} aptlyStatus "Aptly is unavailable"
+// @Router /api/ready [get]
+func apiReady(isReady *atomic.Value) func(*gin.Context) {
+	return func(c *gin.Context) {
+		if isReady == nil || !isReady.Load().(bool) {
+			c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
+			return
+		}
+
+		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+	}
+}
+
+// @Summary Get Health State
+// @Description **Get aptly health state**
+// @Description
+// @Description Return aptly health state:
+// @Description - `Aptly is healthy` (HTTP 200)
+// @Tags Status
+// @Produce json
+// @Success 200 {object} aptlyStatus
+// @Router /api/healthy [get]
+func apiHealthy(c *gin.Context) {
+	c.JSON(200, gin.H{"Status": "Aptly is healthy"})
+}
+
 type dbRequestKind int
 
 const (
@@ -35,49 +96,14 @@ type dbRequest struct {
 	err  chan<- error
 }
 
-// Flushes all collections which cache in-memory objects
-func flushColections() {
-	// lock everything to eliminate in-progress calls
-	r := context.CollectionFactory().RemoteRepoCollection()
-	r.Lock()
-	defer r.Unlock()
-
-	l := context.CollectionFactory().LocalRepoCollection()
-	l.Lock()
-	defer l.Unlock()
-
-	s := context.CollectionFactory().SnapshotCollection()
-	s.Lock()
-	defer s.Unlock()
-
-	p := context.CollectionFactory().PublishedRepoCollection()
-	p.Lock()
-	defer p.Unlock()
-
-	// all collections locked, flush them
-	context.CollectionFactory().Flush()
-}
-
-// Periodically flushes CollectionFactory to free up memory used by
-// collections, flushing caches.
-//
-// Should be run in goroutine!
-func cacheFlusher() {
-	ticker := time.Tick(15 * time.Minute)
-
-	for {
-		<-ticker
-
-		flushColections()
-	}
-}
+var dbRequests chan dbRequest
 
 // Acquire database lock and release it when not needed anymore.
 //
 // Should be run in a goroutine!
-func acquireDatabase(requests <-chan dbRequest) {
+func acquireDatabase() {
 	clients := 0
-	for request := range requests {
+	for request := range dbRequests {
 		var err error
 
 		switch request.kind {
@@ -94,7 +120,6 @@ func acquireDatabase(requests <-chan dbRequest) {
 		case releasedb:
 			clients--
 			if clients == 0 {
-				flushColections()
 				err = context.CloseDatabase()
 			} else {
 				err = nil
@@ -105,14 +130,111 @@ func acquireDatabase(requests <-chan dbRequest) {
 	}
 }
 
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Debug().Msg("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Debug().Msg("Executing task synchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			AbortWithJSONError(c, 409, conflictErr)
+			return
+		}
+
+		// wait for task to finish
+		context.TaskList().WaitForTaskByID(task.ID)
+
+		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
+		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
+		context.TaskList().DeleteTaskByID(task.ID)
+		if err != nil {
+			AbortWithJSONError(c, retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
+	}
+}
+
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
-		c.Fail(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
@@ -120,7 +242,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.Fail(400, err)
+			AbortWithJSONError(c, 400, err)
 			return
 		}
 
@@ -137,21 +259,57 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
 
 		list.PrepareIndex()
 
-		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
-			nil, context.DependencyOptions(), architecturesList)
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			Source:            nil,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
 		if err != nil {
-			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}
 
+	// filter packages by version
+	if c.Request.URL.Query().Get("maximumVersion") == "1" {
+		list.PrepareIndex()
+		list.ForEach(func(p *deb.Package) error {
+			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
+			if err != nil {
+				fmt.Println("filter packages by version, query string parse err: ", err)
+				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+			} else {
+				tmpList, err := list.Filter(deb.FilterOptions{
+					Queries: []deb.PackageQuery{versionQ},
+				})
+
+				if err == nil {
+					if tmpList.Len() > 0 {
+						tmpList.ForEach(func(tp *deb.Package) error {
+							list.Remove(tp)
+							return nil
+						})
+						list.Add(p)
+					}
+				} else {
+					fmt.Println("filter packages by version, filter err: ", err)
+					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+				}
+			}
+
+			return nil
+		})
+	}
+
 	if c.Request.URL.Query().Get("format") == "details" {
 		list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
@@ -163,3 +321,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		c.JSON(200, list.Strings())
 	}
 }
+
+func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
+	return c.AbortWithError(code, err)
+}

api/api_test.go

@@ -0,0 +1,175 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ApiSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+}
+
+var _ = Suite(&ApiSuite{})
+
+func createTestConfig() *os.File {
+	file, err := os.CreateTemp("", "aptly")
+	if err != nil {
+		return nil
+	}
+	jsonString, err := json.Marshal(gin.H{
+		"architectures":         []string{},
+		"enableMetricsEndpoint": true,
+	})
+	if err != nil {
+		return nil
+	}
+	file.Write(jsonString)
+	return file
+}
+
+func (s *ApiSuite) setupContext() error {
+	aptly.Version = "testVersion"
+	file := createTestConfig()
+	if nil == file {
+		return fmt.Errorf("unable to create the test configuration file")
+	}
+	s.configFile = file
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	if nil != err {
+		return err
+	}
+
+	s.context = context
+	s.router = Router(context)
+
+	return nil
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	err := s.setupContext()
+	c.Assert(err, IsNil)
+}
+
+func (s *ApiSuite) TearDownSuite(c *C) {
+	os.Remove(s.configFile.Name())
+	s.context.Shutdown()
+}
+
+func (s *ApiSuite) SetUpTest(c *C) {
+}
+
+func (s *ApiSuite) TearDownTest(c *C) {
+}
+
+func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w, nil
+}
+
+func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
+}
+
+func (s *ApiSuite) TestGetVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/version", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
+}
+
+func (s *ApiSuite) TestGetReadiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/ready", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
+}
+
+func (s *ApiSuite) TestGetHealthiness(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
+}
+
+func (s *ApiSuite) TestGetMetrics(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	b := strings.Replace(response.Body.String(), "\n", "", -1)
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
+	c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
+	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
+}
+
+func (s *ApiSuite) TestRepoCreate(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name": "dummy",
+	})
+	c.Assert(err, IsNil)
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+}
+
+func (s *ApiSuite) TestTruthy(c *C) {
+	c.Check(truthy("no"), Equals, false)
+	c.Check(truthy("n"), Equals, false)
+	c.Check(truthy("off"), Equals, false)
+	c.Check(truthy("false"), Equals, false)
+	c.Check(truthy("0"), Equals, false)
+	c.Check(truthy(false), Equals, false)
+	c.Check(truthy(0), Equals, false)
+
+	c.Check(truthy("y"), Equals, true)
+	c.Check(truthy("yes"), Equals, true)
+	c.Check(truthy("t"), Equals, true)
+	c.Check(truthy("true"), Equals, true)
+	c.Check(truthy("1"), Equals, true)
+	c.Check(truthy(true), Equals, true)
+	c.Check(truthy(1), Equals, true)
+
+	c.Check(truthy(nil), Equals, false)
+
+	c.Check(truthy("foobar"), Equals, true)
+	c.Check(truthy(-1), Equals, true)
+	c.Check(truthy(gin.H{}), Equals, true)
+}

api/db.go

@@ -0,0 +1,188 @@
+package api
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary DB Cleanup
+// @Description **Cleanup Aptly DB**
+// @Description Database cleanup removes information about unreferenced packages and deletes files in the package pool that aren’t used by packages anymore.
+// @Description It is a good idea to run this command after massive deletion of mirrors, snapshots or local repos.
+// @Tags Database
+// @Produce json
+// @Param _async query bool false "Run in background and return task object"
+// @Success 200 {object} string "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/db/cleanup [post]
+func apiDbCleanup(c *gin.Context) {
+	resources := []string{string(task.AllResourcesKey)}
+	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var err error
+
+		collectionFactory := context.NewCollectionFactory()
+
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
+
+		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
+		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if e != nil {
+				return e
+			}
+
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+			if published.SourceKind != deb.SourceLocalRepo {
+				return nil
+			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
+
+			for _, component := range published.Components() {
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// ... and compare it to the list of all packages
+		out.Printf("Loading list of all packages...")
+		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+		// delete packages that are no longer referenced
+		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
+
+		// database can't err as collection factory already constructed
+		db, _ := context.Database()
+
+		if toDelete.Len() > 0 {
+			batch := db.CreateBatch()
+			toDelete.ForEach(func(ref []byte) error {
+				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				return nil
+			})
+
+			err = batch.Write()
+			if err != nil {
+				return nil, fmt.Errorf("unable to write to DB: %s", err)
+			}
+		}
+
+		// now, build a list of files that should be present in Repository (package pool)
+		out.Printf("Building list of files referenced by packages...")
+		referencedFiles := make([]string, 0, existingPackageRefs.Len())
+
+		err = existingPackageRefs.ForEach(func(key []byte) error {
+			pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+			if err2 != nil {
+				tail := ""
+				return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+			}
+			paths, err2 := pkg.FilepathList(context.PackagePool())
+			if err2 != nil {
+				return err2
+			}
+			referencedFiles = append(referencedFiles, paths...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sort.Strings(referencedFiles)
+
+		// build a list of files in the package pool
+		out.Printf("Building list of files in package pool...")
+		existingFiles, err := context.PackagePool().FilepathList(out)
+		if err != nil {
+			return nil, fmt.Errorf("unable to collect file paths: %s", err)
+		}
+
+		// find files which are in the pool but not referenced by packages
+		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+		// delete files that are no longer referenced
+		out.Printf("Deleting unreferenced files (%d)...", len(filesToDelete))
+
+		countFilesToDelete := len(filesToDelete)
+		taskDetail := struct {
+			TotalNumberOfPackagesToDelete     int
+			RemainingNumberOfPackagesToDelete int
+		}{
+			countFilesToDelete, countFilesToDelete,
+		}
+		detail.Store(taskDetail)
+
+		if countFilesToDelete > 0 {
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return nil, err
+				}
+
+				taskDetail.RemainingNumberOfPackagesToDelete--
+				detail.Store(taskDetail)
+				totalSize += size
+			}
+
+			out.Printf("Disk space freed: %s...", utils.HumanBytes(totalSize))
+		}
+
+		out.Printf("Compacting database...")
+		return nil, db.CompactDB()
+	})
+}

api/error.go

@@ -0,0 +1,5 @@
+package api
+
+type Error struct {
+	Error string `json:"error"`
+}

api/files.go

@@ -6,8 +6,11 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
+	"github.com/saracen/walker"
 )
 
 func verifyPath(path string) bool {
@@ -24,27 +27,37 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.Fail(400, fmt.Errorf("wrong dir"))
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
 	return true
 }
 
-// GET /files
+// @Summary List Directories
+// @Description **Get list of upload directories**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description  $ curl http://localhost:8080/api/files
+// @Description  ["aptly-0.9"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Success 200 {array} string "List of files"
+// @Router /api/files [get]
 func apiFilesListDirs(c *gin.Context) {
 	list := []string{}
+	listLock := &sync.Mutex{}
 
-	err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
+	err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
 		if path == context.UploadPath() {
 			return nil
 		}
 
 		if info.IsDir() {
+			listLock.Lock()
+			defer listLock.Unlock()
 			list = append(list, filepath.Base(path))
 			return filepath.SkipDir
 		}
@@ -53,30 +66,50 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.Fail(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
 	c.JSON(200, list)
 }
 
-// POST /files/:dir/
+// @Summary Upload Files
+// @Description **Upload files to a directory**
+// @Description
+// @Description - one or more files can be uploaded
+// @Description - existing uploaded are overwritten
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X POST -F file=@aptly_0.9~dev+217+ge5d646c_i386.deb http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Accept multipart/form-data
+// @Param dir path string true "Directory to upload files to. Created if does not exist"
+// @Param files formData file true "Files to upload"
+// @Produce json
+// @Success 200 {array} string "list of uploaded files"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [post]
 func apiFilesUpload(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.Fail(400, err)
+		AbortWithJSONError(c, 400, err)
 		return
 	}
 
@@ -86,7 +119,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +127,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
@@ -109,20 +142,35 @@ func apiFilesUpload(c *gin.Context) {
 		}
 	}
 
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
-
 }
 
-// GET /files/:dir
+// @Summary List Files
+// @Description **Show uploaded files in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl http://localhost:8080/api/files/aptly-0.9
+// @Description ["aptly_0.9~dev+217+ge5d646c_i386.deb"]
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to list"
+// @Success 200 {array} string "Files found in directory"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [get]
 func apiFilesListFiles(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
 	list := []string{}
-	root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
+	listLock := &sync.Mutex{}
+	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -131,6 +179,8 @@ func apiFilesListFiles(c *gin.Context) {
 			return nil
 		}
 
+		listLock.Lock()
+		defer listLock.Unlock()
 		list = append(list, filepath.Base(path))
 
 		return nil
@@ -138,9 +188,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.Fail(404, err)
+			AbortWithJSONError(c, 404, err)
 		} else {
-			c.Fail(500, err)
+			AbortWithJSONError(c, 500, err)
 		}
 		return
 	}
@@ -148,36 +198,66 @@ func apiFilesListFiles(c *gin.Context) {
 	c.JSON(200, list)
 }
 
-// DELETE /files/:dir
+// @Summary Delete Directory
+// @Description **Delete upload directory and uploaded files within**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir} [delete]
 func apiFilesDeleteDir(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	c.JSON(200, gin.H{})
 }
 
-// DELETE /files/:dir/:name
+// @Summary Delete File
+// @Description **Delete a uploaded file in upload directory**
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ curl -X DELETE http://localhost:8080/api/files/aptly-0.9/aptly_0.9~dev+217+ge5d646c_i386.deb
+// @Description {}
+// @Description  ```
+// @Tags Files
+// @Produce json
+// @Param dir path string true "Directory to delete from"
+// @Param name path string true "File to delete"
+// @Success 200 {object} string "msg"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir}/{name} [delete]
 func apiFilesDeleteFile(c *gin.Context) {
 	if !verifyDir(c) {
 		return
 	}
 
-	if !verifyPath(c.Params.ByName("name")) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+	dir := utils.SanitizePath(c.Params.ByName("dir"))
+	name := utils.SanitizePath(c.Params.ByName("name"))
+	if !verifyPath(name) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
+	err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.Fail(500, err)
+			AbortWithJSONError(c, 500, err)
 			return
 		}
 	}

api/gpg.go

@@ -0,0 +1,102 @@
+package api
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+type gpgAddKeyParams struct {
+	// Keyserver, when downloading GpgKeyIDs
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// GpgKeyIDs to download from Keyserver, comma separated list
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
+	// Armored gpg public ket, instead of downloading from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+	// Keyring for adding the keys (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+}
+
+// @Summary Add GPG Keys
+// @Description **Adds GPG keys to aptly keyring**
+// @Description
+// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/gpg [post]
+func apiGPGAddKey(c *gin.Context) {
+	b := gpgAddKeyParams{}
+	if c.Bind(&b) != nil {
+		return
+	}
+	b.Keyserver = utils.SanitizePath(b.Keyserver)
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
+	// b.Keyring can be an absolute path
+
+	var err error
+	args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+	args = append(args, "--keyring", keyring)
+	if len(b.Keyserver) > 0 {
+		args = append(args, "--keyserver", b.Keyserver)
+	}
+	if len(b.GpgKeyArmor) > 0 {
+		var tempdir string
+		tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+		defer os.RemoveAll(tempdir)
+
+		keypath := filepath.Join(tempdir, "key")
+		keyfile, e := os.Create(keypath)
+		if e != nil {
+			AbortWithJSONError(c, 400, e)
+			return
+		}
+		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
+			AbortWithJSONError(c, 400, e)
+		}
+		args = append(args, "--import", keypath)
+
+	}
+	if len(b.GpgKeyID) > 0 {
+		keys := strings.Fields(b.GpgKeyID)
+		args = append(args, "--recv-keys")
+		args = append(args, keys...)
+	}
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	// it might happened that we have a situation with an erroneous
+	// gpg command (e.g. when GpgKeyID and GpgKeyArmor is set).
+	// there is no error handling for such as gpg will do this for us
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		c.JSON(400, string(out))
+		return
+	}
+
+	c.JSON(200, string(out))
+}

api/graph.go

@@ -8,11 +8,31 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/deb"
 )
 
-// GET /api/graph.:ext?layout=[vertical|horizontal(default)]
+// @Summary Graph Output
+// @Description **Generate dependency graph**
+// @Description
+// @Description Command graph generates graph of dependencies:
+// @Description
+// @Description * between snapshots and mirrors (what mirror was used to create each snapshot)
+// @Description * between snapshots and local repos (what local repo was used to create snapshot)
+// @Description * between snapshots (pulling, merging, etc.)
+// @Description * between snapshots, local repos and published repositories (how snapshots were published).
+// @Description
+// @Description Graph is rendered to PNG file using graphviz package.
+// @Description
+// @Description Example URL: `http://localhost:8080/api/graph.svg?layout=vertical`
+// @Tags Status
+// @Produce image/png, image/svg+xml
+// @Param ext path string true "ext specifies desired file extension, e.g. .png, .svg."
+// @Param layout query string false "Change between a `horizontal` (default) and a `vertical` graph layout."
+// @Success 200 {object} []byte "Output"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/graph.{ext} [get]
 func apiGraph(c *gin.Context) {
 	var (
 		err    error
@@ -21,17 +41,7 @@ func apiGraph(c *gin.Context) {
 
 	ext := c.Params.ByName("ext")
 	layout := c.Request.URL.Query().Get("layout")
-
-	factory := context.CollectionFactory()
-
-	factory.RemoteRepoCollection().RLock()
-	defer factory.RemoteRepoCollection().RUnlock()
-	factory.LocalRepoCollection().RLock()
-	defer factory.LocalRepoCollection().RUnlock()
-	factory.SnapshotCollection().RLock()
-	defer factory.SnapshotCollection().RUnlock()
-	factory.PublishedRepoCollection().RLock()
-	defer factory.PublishedRepoCollection().RUnlock()
+	factory := context.NewCollectionFactory()
 
 	graph, err := deb.BuildGraph(factory, layout)
 	if err != nil {
@@ -53,25 +63,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.Fail(500, err)
+		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/metrics.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/rs/zerolog/log"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiVersionGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_build_info",
+			Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
+		},
+		[]string{"version", "goversion"},
+	)
+	apiFilesUploadedCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_files_uploaded_total",
+			Help: "Total number of uploaded files labeled by upload directory.",
+		},
+		[]string{"directory"},
+	)
+	apiReposPackageCountGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_repos_package_count",
+			Help: "Current number of published packages labeled by source, distribution and component.",
+		},
+		[]string{"source", "distribution", "component"},
+	)
+)
+
+type metricsCollectorRegistrar struct {
+	hasRegistered bool
+}
+
+func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
+	if !r.hasRegistered {
+		apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+		r.hasRegistered = true
+	}
+}
+
+var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
+
+func countPackagesByRepos() {
+	err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if err != nil {
+			msg := fmt.Sprintf(
+				"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
+				err, repo.StoragePrefix(), repo.Distribution, repo.Components())
+			log.Warn().Msg(msg)
+			return err
+		}
+
+		components := repo.Components()
+		for _, c := range components {
+			count := float64(len(repo.RefList(c).Refs))
+			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
+		log.Warn().Msg(msg)
+	}
+}

api/middleware.go

@@ -0,0 +1,116 @@
+package api
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/rs/zerolog/log"
+)
+
+// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
+// See https://prometheus.io/docs/practices/naming/#labels
+func getBasePath(c *gin.Context) string {
+	segment0, err := getURLSegment(c.Request.URL.Path, 0)
+	if err != nil {
+		return "/"
+	}
+	segment1, err := getURLSegment(c.Request.URL.Path, 1)
+	if err != nil {
+		return *segment0
+	}
+
+	return *segment0 + *segment1
+}
+
+func getURLSegment(url string, idx int) (*string, error) {
+	urlSegments := strings.Split(url, "/")
+	// Remove segment at index 0 because it's an empty string
+	urlSegments = urlSegments[1:cap(urlSegments)]
+
+	if len(urlSegments) <= idx {
+		return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
+	}
+
+	segmentAtIndex := urlSegments[idx]
+	s := fmt.Sprintf("/%s", segmentAtIndex)
+	return &s, nil
+}
+
+func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		g.WithLabelValues(c.Request.Method, pathFunc(c)).Inc()
+		defer g.WithLabelValues(c.Request.Method, pathFunc(c)).Dec()
+		c.Next()
+	}
+}
+
+func instrumentHandlerCounter(counter *prometheus.CounterVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		counter.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Inc()
+	}
+}
+
+func instrumentHandlerRequestSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(float64(c.Request.ContentLength))
+	}
+}
+
+func instrumentHandlerResponseSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		var responseSize = math.Max(float64(c.Writer.Size()), 0)
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(responseSize)
+	}
+}
+
+func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		now := time.Now()
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
+	}
+}
+
+// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
+// logs that include error messages if there are any.
+func JSONLogger() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		// Start timer
+		start := time.Now()
+		path := c.Request.URL.Path
+		raw := c.Request.URL.RawQuery
+
+		// Process request
+		c.Next()
+
+		ts := time.Now()
+		if raw != "" {
+			path = path + "?" + raw
+		}
+
+		errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
+		l := log.With().Str("remote", c.ClientIP()).Logger().
+			With().Str("method", c.Request.Method).Logger().
+			With().Str("path", path).Logger().
+			With().Str("protocol", c.Request.Proto).Logger().
+			With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
+			With().Str("latency", ts.Sub(start).String()).Logger().
+			With().Str("agent", c.Request.UserAgent()).Logger()
+
+		if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
+			l.Warn().Msg(errorMessage)
+		} else if c.Writer.Status() >= 500 {
+			l.Error().Msg(errorMessage)
+		} else {
+			l.Info().Msg(errorMessage)
+		}
+	}
+}

api/middleware_test.go

@@ -0,0 +1,255 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"sync/atomic"
+
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MiddlewareSuite struct {
+	router    http.Handler
+	context   *gin.Context
+	logReader *os.File
+	logWriter *os.File
+}
+
+var _ = Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) SetUpTest(c *C) {
+	r, w, err := os.Pipe()
+	c.Assert(err, IsNil)
+
+	utils.SetupJSONLogger("debug", w)
+	mw := JSONLogger()
+
+	router := gin.New()
+	router.UseRawPath = true
+	router.Use(mw)
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	root := router.Group("/api")
+	isReady := &atomic.Value{}
+	isReady.Store(false)
+	root.GET("/ready", apiReady(isReady))
+	root.GET("/healthy", apiHealthy)
+
+	s.router = router
+	s.logReader = r
+	s.logWriter = w
+}
+
+func (s *MiddlewareSuite) TearDownTest(c *C) {
+	s.router = nil
+	s.context = nil
+	s.logReader = nil
+	s.logWriter = nil
+}
+
+func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
+	recorder := httptest.NewRecorder()
+	s.context, _ = gin.CreateTestContext(recorder)
+	req, _ := http.NewRequestWithContext(s.context, method, url, body)
+	s.context.Request = req
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(httptest.NewRecorder(), req)
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "warn")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["method"]; ok {
+		c.Check(val, Equals, "GET")
+	} else {
+		c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["path"]; ok {
+		c.Check(val, Equals, "/")
+	} else {
+		c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["protocol"]; ok {
+		c.Check(val, Equals, "HTTP/1.1")
+	} else {
+		c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
+	}
+
+	if val, ok := jsonMap["code"]; ok {
+		c.Check(val, Equals, "404")
+	} else {
+		c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["remote"]; !ok {
+		c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["latency"]; !ok {
+		c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["agent"]; !ok {
+		c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
+	}
+
+	if _, ok := jsonMap["time"]; !ok {
+		c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "error")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
+	outC := make(chan string)
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, s.logReader)
+		fmt.Println(buf.String())
+		outC <- buf.String()
+	}()
+
+	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
+	s.logWriter.Close()
+	capturedOutput := <-outC
+
+	var jsonMap map[string]interface{}
+	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+
+	fmt.Println(capturedOutput)
+
+	if val, ok := jsonMap["level"]; ok {
+		c.Check(val, Equals, "info")
+	} else {
+		c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
+	}
+}
+
+func (s *MiddlewareSuite) TestGetBasePath(c *C) {
+	s.HTTPRequest(http.MethodGet, "", nil)
+	path := getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/")
+
+	s.HTTPRequest(http.MethodGet, "/api", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api")
+
+	s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
+	path = getBasePath(s.context)
+	c.Check(path, Equals, "/api/repos")
+}
+
+func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
+	url := "/"
+	segment, err := getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	_, err = getURLSegment(url, 1)
+	if err == nil {
+		c.Error("Invalid return value")
+	}
+
+	url = "/api/repos/testRepo"
+	segment, err = getURLSegment(url, 0)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/api")
+
+	segment, err = getURLSegment(url, 1)
+	if err != nil {
+		c.Error(err)
+	}
+	c.Check(*segment, Equals, "/repos")
+}

api/mirror.go

@@ -0,0 +1,665 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+	"github.com/rs/zerolog/log"
+)
+
+func getVerifier(keyRings []string) (pgp.Verifier, error) {
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring(false)
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+// @Summary List Mirrors
+// @Description **Show list of currently available mirrors**
+// @Description Each mirror is returned as in “show” API.
+// @Tags Mirrors
+// @Produce json
+// @Success 200 {array} deb.RemoteRepo
+// @Router /api/mirrors [get]
+func apiMirrorsList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	result := []*deb.RemoteRepo{}
+	collection.ForEach(func(repo *deb.RemoteRepo) error {
+		result = append(result, repo)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+type mirrorCreateParams struct {
+	// Name of mirror to be created
+	Name string `binding:"required"          json:"Name"              example:"mirror2"`
+	// Url of the archive to mirror
+	ArchiveURL string `binding:"required"    json:"ArchiveURL"        example:"http://deb.debian.org/debian"`
+	// Distribution name to mirror
+	Distribution string `                    json:"Distribution"      example:"'buster', for flat repositories use './'"`
+	// Package query that is applied to mirror packages
+	Filter string `                          json:"Filter"            example:"xserver-xorg"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `                    json:"Components"        example:"main"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `                 json:"Architectures"     example:"amd64"`
+	// Gpg keyring(s) for verifying Release file
+	Keyrings []string `                      json:"Keyrings"          example:"trustedkeys.gpg"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `                   json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `                     json:"DownloadUdebs"`
+	// Set "true" to mirror installer files
+	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `                    json:"FilterWithDeps"`
+	// Set "true" to skip if the given components are in the Release file
+	SkipComponentCheck bool `                json:"SkipComponentCheck"`
+	// Set "true" to skip the verification of architectures
+	SkipArchitectureCheck bool `             json:"SkipArchitectureCheck"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `                  json:"IgnoreSignatures"`
+}
+
+// @Summary Create Mirror
+// @Description **Create a mirror of a remote repository**
+// @Tags Mirrors
+// @Consume json
+// @Param request body mirrorCreateParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/mirrors [post]
+func apiMirrorsCreate(c *gin.Context) {
+	var err error
+	var b mirrorCreateParams
+
+	b.DownloadSources = context.Config().DownloadSourcePackages
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+	b.Architectures = context.ArchitecturesList()
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
+		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+	}
+
+	if b.Filter != "" {
+		_, err = query.Parse(b.Filter)
+		if err != nil {
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
+			return
+		}
+	}
+
+	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
+		return
+	}
+
+	repo.Filter = b.Filter
+	repo.FilterWithDeps = b.FilterWithDeps
+	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.SkipArchitectureCheck = b.SkipArchitectureCheck
+	repo.DownloadSources = b.DownloadSources
+	repo.DownloadUdebs = b.DownloadUdebs
+
+	verifier, err := getVerifier(b.Keyrings)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	downloader := context.NewDownloader(nil)
+	err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
+		return
+	}
+
+	err = collection.Add(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// @Summary Delete Mirror
+// @Description **Delete a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param force query int true "force: 1 to enable"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
+// @Failure 500 {object} Error "Unable to delete"
+// @Router /api/mirrors/{name} [delete]
+func apiMirrorsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	mirrorCollection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	repo, err := mirrorCollection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+			}
+		}
+
+		err = mirrorCollection.Drop(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}
+
+// @Summary Get Mirror Info
+// @Description **Get mirror information by name**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [get]
+func apiMirrorsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	c.JSON(200, repo)
+}
+
+// @Summary List Mirror Packages
+// @Description **Get a list of packages from a mirror**
+// @Tags Mirrors
+// @Param name path string true "mirror name"
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Produce json
+// @Success 200 {array} deb.Package "List of Packages"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name}/packages [get]
+func apiMirrorsPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	if repo.LastDownloadDate.IsZero() {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		return
+	}
+
+	reflist := repo.RefList()
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			AbortWithJSONError(c, 400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter(deb.FilterOptions{
+			Queries:           []deb.PackageQuery{q},
+			WithDependencies:  withDeps,
+			DependencyOptions: context.DependencyOptions(),
+			Architectures:     architecturesList,
+		})
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}
+
+type mirrorUpdateParams struct {
+	// Change mirror name to `Name`
+	Name string `                 json:"Name"                   example:"mirror1"`
+	// Url of the archive to mirror
+	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
+	// Package query that is applied to mirror packages
+	Filter string `               json:"Filter"                 example:"xserver-xorg"`
+	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
+	Architectures []string `      json:"Architectures"          example:"amd64"`
+	// Components to mirror, if not specified aptly would fetch all components
+	Components []string `         json:"Components"             example:"main"`
+	// Gpg keyring(s) for verifing Release file
+	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps bool `         json:"FilterWithDeps"`
+	// Set "true" to mirror source packages
+	DownloadSources bool `        json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs bool `          json:"DownloadUdebs"`
+	// Set "true" to skip checking if the given components are in the Release file
+	SkipComponentCheck bool `     json:"SkipComponentCheck"`
+	// Set "true" to skip checking if the given architectures are in the Release file
+	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
+	// Set "true" to ignore checksum errors
+	IgnoreChecksums bool `        json:"IgnoreChecksums"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures bool `       json:"IgnoreSignatures"`
+	// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
+	ForceUpdate bool `            json:"ForceUpdate"`
+	// Set "true" to skip downloading already downloaded packages
+	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+}
+
+// @Summary Update Mirror
+// @Description **Update Mirror and download packages**
+// @Tags Mirrors
+// @Param name path string true "mirror name to update"
+// @Consume json
+// @Param request body mirrorUpdateParams true "Parameters"
+// @Param _async query bool false "Run in background and return task object"
+// @Produce json
+// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
+// @Success 202 {object} task.Task "Mirror is being updated"
+// @Failure 400 {object} Error "Unable to determine list of architectures"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [put]
+func apiMirrorsUpdate(c *gin.Context) {
+	var (
+		err    error
+		remote *deb.RemoteRepo
+		b      mirrorUpdateParams
+	)
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	remote, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	b.Name = remote.Name
+	b.DownloadUdebs = remote.DownloadUdebs
+	b.DownloadSources = remote.DownloadSources
+	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
+	b.FilterWithDeps = remote.FilterWithDeps
+	b.Filter = remote.Filter
+	b.Architectures = remote.Architectures
+	b.Components = remote.Components
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+
+	log.Info().Msgf("%s: Starting mirror update", b.Name)
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Name != remote.Name {
+		_, err = collection.ByName(b.Name)
+		if err == nil {
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			return
+		}
+	}
+
+	if b.DownloadUdebs != remote.DownloadUdebs {
+		if remote.IsFlat() && b.DownloadUdebs {
+			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			return
+		}
+	}
+
+	if b.ArchiveURL != "" {
+		remote.SetArchiveRoot(b.ArchiveURL)
+	}
+
+	remote.Name = b.Name
+	remote.DownloadUdebs = b.DownloadUdebs
+	remote.DownloadSources = b.DownloadSources
+	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
+	remote.FilterWithDeps = b.FilterWithDeps
+	remote.Filter = b.Filter
+	remote.Architectures = b.Architectures
+	remote.Components = b.Components
+
+	verifier, err := getVerifier(b.Keyrings)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	resources := []string{string(remote.Key())}
+	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+		downloader := context.NewDownloader(out)
+		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if !b.ForceUpdate {
+			err = remote.CheckLock()
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if remote.Filter != "" {
+			var filterQuery deb.PackageQuery
+
+			filterQuery, err = query.Parse(remote.Filter)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			_, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		defer func() {
+			// on any interruption, unlock the mirror
+			e := context.ReOpenDatabase()
+			if e == nil {
+				remote.MarkAsIdle()
+				collection.Update(remote)
+			}
+		}()
+
+		remote.MarkAsUpdating()
+		err = collection.Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		context.GoContextHandleSignals()
+
+		count := len(queue)
+		taskDetail := struct {
+			TotalDownloadSize         int64
+			RemainingDownloadSize     int64
+			TotalNumberOfPackages     int
+			RemainingNumberOfPackages int
+		}{
+			downloadSize, downloadSize, count, count,
+		}
+		detail.Store(taskDetail)
+
+		downloadQueue := make(chan int)
+		taskFinished := make(chan *deb.PackageDownloadTask)
+
+		var (
+			errors  []string
+			errLock sync.Mutex
+		)
+
+		pushError := func(err error) {
+			errLock.Lock()
+			errors = append(errors, err.Error())
+			errLock.Unlock()
+		}
+
+		go func() {
+			for idx := range queue {
+				select {
+				case downloadQueue <- idx:
+				case <-context.Done():
+					return
+				}
+			}
+
+			close(downloadQueue)
+		}()
+
+		// update of task details need to be done in order
+		go func() {
+			for {
+				task, ok := <-taskFinished
+				if !ok {
+					return
+				}
+
+				taskDetail.RemainingDownloadSize -= task.File.Checksums.Size
+				taskDetail.RemainingNumberOfPackages--
+				detail.Store(taskDetail)
+			}
+		}()
+
+		log.Info().Msgf("%s: Spawning background processes...", b.Name)
+		var wg sync.WaitGroup
+		for i := 0; i < context.Config().DownloadConcurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case idx, ok := <-downloadQueue:
+						if !ok {
+							return
+						}
+
+						task := &queue[idx]
+
+						var e error
+
+						// provision download location
+						if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+							task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+						} else {
+							var file *os.File
+							file, e = os.CreateTemp("", task.File.Filename)
+							if e == nil {
+								task.TempDownPath = file.Name()
+								file.Close()
+							}
+						}
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// download file...
+						e = context.Downloader().DownloadWithChecksum(
+							context,
+							remote.PackageURL(task.File.DownloadURL()).String(),
+							task.TempDownPath,
+							&task.File.Checksums,
+							b.IgnoreChecksums)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// and import it back to the pool
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						if err != nil {
+							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+							pushError(err)
+							continue
+						}
+
+						// update "attached" files if any
+						for _, additionalAtask := range task.Additional {
+							additionalAtask.File.PoolPath = task.File.PoolPath
+							additionalAtask.File.Checksums = task.File.Checksums
+						}
+
+						task.Done = true
+						taskFinished <- task
+					case <-context.Done():
+						return
+					}
+
+				}
+			}()
+		}
+
+		// Wait for all download goroutines to finish
+		log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
+		wg.Wait()
+		log.Info().Msgf("%s: Background processes finished", b.Name)
+		close(taskFinished)
+
+		defer func() {
+			for _, task := range queue {
+				if task.TempDownPath == "" {
+					continue
+				}
+
+				if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+					fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+				}
+			}
+		}()
+
+		select {
+		case <-context.Done():
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+
+		if len(errors) > 0 {
+			log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+		}
+
+		log.Info().Msgf("%s: Finalizing download...", b.Name)
+		remote.FinalizeDownload(collectionFactory, out)
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		log.Info().Msgf("%s: Mirror updated successfully", b.Name)
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}

api/mirror_test.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MirrorSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&MirrorSuite{})
+
+func (s *MirrorSuite) TestGetMirrors(c *C) {
+	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}
+
+func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
+	response, _ := s.HTTPRequest("DELETE", "/api/mirrors/does-not-exist", nil)
+	c.Check(response.Code, Equals, 404)
+	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
+}
+
+func (s *MirrorSuite) TestCreateMirror(c *C) {
+	c.ExpectFailure("Need to mock downloads")
+	body, err := json.Marshal(gin.H{
+		"Name":       "dummy",
+		"ArchiveURL": "foobar",
+	})
+	c.Assert(err, IsNil)
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Equals, "")
+}

api/packages.go

@@ -1,16 +1,41 @@
 package api
 
 import (
+	_ "github.com/aptly-dev/aptly/deb" // for swagger
 	"github.com/gin-gonic/gin"
 )
 
-// GET /api/packages/:key
+// @Summary Get Package Info
+// @Description **Show information about package by package key**
+// @Description Package keys could be obtained from various GET .../packages APIs.
+// @Tags Packages
+// @Produce json
+// @Param key path string true "package key (unique package identifier)"
+// @Success 200 {object} deb.Package "OK"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/packages/{key} [get]
 func apiPackagesShow(c *gin.Context) {
-	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	collectionFactory := context.NewCollectionFactory()
+	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.Fail(404, err)
+		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	c.JSON(200, p)
 }
+
+// @Summary List Packages
+// @Description **Get list of packages**
+// @Tags Packages
+// @Consume  json
+// @Produce  json
+// @Param q query string false "search query"
+// @Param format query string false "format: `details` for more detailed information"
+// @Success 200 {array} string "List of packages"
+// @Router /api/packages [get]
+func apiPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PackageCollection()
+	showPackages(c, collection.AllPackageRefs(), collectionFactory)
+}

api/packages_test.go

@@ -0,0 +1,18 @@
+package api
+
+import (
+	. "gopkg.in/check.v1"
+)
+
+type PackagesSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&PackagesSuite{})
+
+func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}

api/router.go

@@ -2,114 +2,236 @@ package api
 
 import (
 	"net/http"
+	"os"
+	"sync/atomic"
 
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
-	ctx "github.com/smira/aptly/context"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/rs/zerolog/log"
+
+	"github.com/aptly-dev/aptly/docs"
+	swaggerFiles "github.com/swaggo/files"
+	ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+func apiMetricsGet() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		countPackagesByRepos()
+		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
+	}
+}
+
+func redirectSwagger(c *gin.Context) {
+	if c.Request.URL.Path == "/docs/index.html" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs/" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	if c.Request.URL.Path == "/docs" {
+		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+		return
+	}
+	c.Next()
+}
+
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
+	if aptly.EnableDebug {
+		gin.SetMode(gin.DebugMode)
+	} else {
+		gin.SetMode(gin.ReleaseMode)
+	}
+
+	router := gin.New()
 	context = c
 
-	router := gin.Default()
-	router.Use(gin.ErrorLogger())
+	router.UseRawPath = true
 
+	if c.Config().LogFormat == "json" {
+		c.StructuredLogging(true)
+		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
+		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
+		router.Use(JSONLogger())
+	} else {
+		c.StructuredLogging(false)
+		utils.SetupDefaultLogger(c.Config().LogLevel)
+		router.Use(gin.Logger())
+	}
+
+	router.Use(gin.Recovery(), gin.ErrorLogger())
+
+	if c.Config().EnableSwaggerEndpoint {
+		router.GET("docs.html", func(c *gin.Context) {
+			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+		})
+		router.Use(redirectSwagger)
+		url := ginSwagger.URL("/docs/doc.json")
+		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	}
+
+	if c.Config().EnableMetricsEndpoint {
+		MetricsCollectorRegistrar.Register(router)
+	}
+
+	if c.Config().ServeInAPIMode {
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
+		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
+	}
+
+	api := router.Group("/api")
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
 		// running, we close the database to free the lock.
-		requests := make(chan dbRequest)
+		dbRequests = make(chan dbRequest)
 
-		go acquireDatabase(requests)
+		go acquireDatabase()
 
-		router.Use(func(c *gin.Context) {
+		api.Use(func(c *gin.Context) {
 			var err error
 
 			errCh := make(chan error)
-			requests <- dbRequest{acquiredb, errCh}
+			dbRequests <- dbRequest{acquiredb, errCh}
 
 			err = <-errCh
 			if err != nil {
-				c.Fail(500, err)
+				AbortWithJSONError(c, 500, err)
 				return
 			}
 
 			defer func() {
-				requests <- dbRequest{releasedb, errCh}
+				dbRequests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.Fail(500, err)
+					AbortWithJSONError(c, 500, err)
 				}
 			}()
 
 			c.Next()
 		})
-
-	} else {
-		go cacheFlusher()
-	}
-
-	root := router.Group("/api")
-
-	{
-		root.GET("/version", apiVersion)
 	}
 
 	{
-		root.GET("/repos", apiReposList)
-		root.POST("/repos", apiReposCreate)
-		root.GET("/repos/:name", apiReposShow)
-		root.PUT("/repos/:name", apiReposEdit)
-		root.DELETE("/repos/:name", apiReposDrop)
+		if c.Config().EnableMetricsEndpoint {
+			api.GET("/metrics", apiMetricsGet())
+		}
+		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)
 
-		root.GET("/repos/:name/packages", apiReposPackagesShow)
-		root.POST("/repos/:name/packages", apiReposPackagesAdd)
-		root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
-
-		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
-		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
-
-		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
+		isReady := &atomic.Value{}
+		isReady.Store(false)
+		defer isReady.Store(true)
+		api.GET("/ready", apiReady(isReady))
+		api.GET("/healthy", apiHealthy)
 	}
 
 	{
-		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
+		api.GET("/repos", apiReposList)
+		api.POST("/repos", apiReposCreate)
+		api.GET("/repos/:name", apiReposShow)
+		api.PUT("/repos/:name", apiReposEdit)
+		api.DELETE("/repos/:name", apiReposDrop)
+
+		api.GET("/repos/:name/packages", apiReposPackagesShow)
+		api.POST("/repos/:name/packages", apiReposPackagesAdd)
+		api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
+
+		api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
+		api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
+		api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
+
+		api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
+		api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 
 	{
-		root.GET("/files", apiFilesListDirs)
-		root.POST("/files/:dir", apiFilesUpload)
-		root.GET("/files/:dir", apiFilesListFiles)
-		root.DELETE("/files/:dir", apiFilesDeleteDir)
-		root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+		api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
 	{
-		root.GET("/publish", apiPublishList)
-		root.POST("/publish", apiPublishRepoOrSnapshot)
-		root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
-		root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
-		root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.GET("/mirrors", apiMirrorsList)
+		api.GET("/mirrors/:name", apiMirrorsShow)
+		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		api.POST("/mirrors", apiMirrorsCreate)
+		api.PUT("/mirrors/:name", apiMirrorsUpdate)
+		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
-		root.GET("/snapshots", apiSnapshotsList)
-		root.POST("/snapshots", apiSnapshotsCreate)
-		root.PUT("/snapshots/:name", apiSnapshotsUpdate)
-		root.GET("/snapshots/:name", apiSnapshotsShow)
-		root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
-		root.DELETE("/snapshots/:name", apiSnapshotsDrop)
-		root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/gpg/key", apiGPGAddKey)
 	}
 
 	{
-		root.GET("/packages/:key", apiPackagesShow)
+		api.GET("/s3", apiS3List)
 	}
 
 	{
-		root.GET("/graph.:ext", apiGraph)
+		api.GET("/files", apiFilesListDirs)
+		api.POST("/files/:dir", apiFilesUpload)
+		api.GET("/files/:dir", apiFilesListFiles)
+		api.DELETE("/files/:dir", apiFilesDeleteDir)
+		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
+	}
+
+	{
+		api.GET("/publish", apiPublishList)
+		api.GET("/publish/:prefix/:distribution", apiPublishShow)
+		api.POST("/publish", apiPublishRepoOrSnapshot)
+		api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
+		api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
+		api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
+		api.POST("/publish/:prefix/:distribution/sources", apiPublishAddSource)
+		api.GET("/publish/:prefix/:distribution/sources", apiPublishListChanges)
+		api.PUT("/publish/:prefix/:distribution/sources", apiPublishSetSources)
+		api.DELETE("/publish/:prefix/:distribution/sources", apiPublishDropChanges)
+		api.PUT("/publish/:prefix/:distribution/sources/:component", apiPublishUpdateSource)
+		api.DELETE("/publish/:prefix/:distribution/sources/:component", apiPublishRemoveSource)
+		api.POST("/publish/:prefix/:distribution/update", apiPublishUpdate)
+	}
+
+	{
+		api.GET("/snapshots", apiSnapshotsList)
+		api.POST("/snapshots", apiSnapshotsCreate)
+		api.PUT("/snapshots/:name", apiSnapshotsUpdate)
+		api.GET("/snapshots/:name", apiSnapshotsShow)
+		api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
+		api.DELETE("/snapshots/:name", apiSnapshotsDrop)
+		api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
+		api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
+		api.POST("/snapshots/:name/pull", apiSnapshotsPull)
+	}
+
+	{
+		api.GET("/packages/:key", apiPackagesShow)
+		api.GET("/packages", apiPackages)
+	}
+
+	{
+		api.GET("/graph.:ext", apiGraph)
+	}
+	{
+		api.POST("/db/cleanup", apiDbCleanup)
+	}
+	{
+		api.GET("/tasks", apiTasksList)
+		api.POST("/tasks-clear", apiTasksClear)
+		api.GET("/tasks-wait", apiTasksWait)
+		api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		api.GET("/tasks/:id/output", apiTasksOutputShow)
+		api.GET("/tasks/:id/detail", apiTasksDetailShow)
+		api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		api.GET("/tasks/:id", apiTasksShow)
+		api.DELETE("/tasks/:id", apiTasksDelete)
 	}
 
 	return router

api/s3.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary S3 buckets
+// @Description **Get list of S3 buckets**
+// @Description
+// @Description List configured S3 buckets.
+// @Tags Status
+// @Produce json
+// @Success 200 {array} string "List of S3 buckets"
+// @Router /api/s3 [get]
+func apiS3List(c *gin.Context) {
+	keys := []string{}
+	for k := range context.Config().S3PublishRoots {
+		keys = append(keys, k)
+	}
+	c.JSON(200, keys)
+}

api/storage.go

@@ -0,0 +1,45 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Description
+// @Description Units in MiB.
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}

api/task.go

@@ -0,0 +1,203 @@
+package api
+
+import (
+	"strconv"
+
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// @Summary List Tasks
+// @Description **Get list of available tasks. Each task is returned as in “show” API**
+// @Tags Tasks
+// @Produce json
+// @Success 200 {array} task.Task
+// @Router /api/tasks [get]
+func apiTasksList(c *gin.Context) {
+	list := context.TaskList()
+	c.JSON(200, list.GetTasks())
+}
+
+// @Summary Clear Tasks
+// @Description **Removes finished and failed tasks from internal task list**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-clear [post]
+func apiTasksClear(c *gin.Context) {
+	list := context.TaskList()
+	list.Clear()
+	c.JSON(200, gin.H{})
+}
+
+// @Summary Wait for all Tasks
+// @Description **Waits for and returns when all running tasks are complete**
+// @Tags Tasks
+// @Produce json
+// @Success 200 ""
+// @Router /api/tasks-wait [get]
+func apiTasksWait(c *gin.Context) {
+	list := context.TaskList()
+	list.Wait()
+	c.JSON(200, gin.H{})
+}
+
+// @Summary Wait for Task
+// @Description **Waits for and returns when given Task ID is complete**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 400 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/wait [get]
+func apiTasksWaitForTaskByID(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	task, err := list.WaitForTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// @Summary Get Task Info
+// @Description **Return task information for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad id?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id} [get]
+func apiTasksShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var task task.Task
+	task, err = list.GetTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// @Summary Get Task Output
+// @Description **Return task output for a given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task output"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/output [get]
+func apiTasksOutputShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var output string
+	output, err = list.GetTaskOutputByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// @Summary Get Task Details
+// @Description **Return task detail for a given ID**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "Task detail"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Task Not Found"
+// @Router /api/tasks/{id}/detail [get]
+func apiTasksDetailShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var detail interface{}
+	detail, err = list.GetTaskDetailByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, detail)
+}
+
+// @Summary Get Task Return Value
+// @Description **Return task return value (status code) by given ID**
+// @Tags Tasks
+// @Produce plain
+// @Param id path int true "Task ID"
+// @Success 200 {object} string "msg"
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 404 {object} Error "Not Found"
+// @Router /api/tasks/{id}/return_value [get]
+func apiTasksReturnValueShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	output, err := list.GetTaskReturnValueByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// @Summary Delete Task
+// @Description **Delete completed task by given ID. Does not stop task execution**
+// @Tags Tasks
+// @Produce json
+// @Param id path int true "Task ID"
+// @Success 200 {object} task.Task
+// @Failure 500 {object} Error "invalid syntax, bad ID?"
+// @Failure 400 {object} Error "Task in progress or not found"
+// @Router /api/tasks/{id} [delete]
+func apiTasksDelete(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	var delTask task.Task
+	delTask, err = list.DeleteTaskByID(int(id))
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	c.JSON(200, delTask)
+}

aptly/conf.go

@@ -0,0 +1,4 @@
+package aptly
+
+// Default aptly.conf (filled in at link time)
+var AptlyConf []byte

aptly/const.go

@@ -0,0 +1,3 @@
+package aptly
+
+var DistributionFocal = "focal"

aptly/interfaces.go

@@ -3,10 +3,12 @@
 package aptly
 
 import (
+	"context"
 	"io"
 	"os"
 
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // ReadSeekerCloser = ReadSeeker + Closer
@@ -33,8 +35,8 @@ type PackagePool interface {
 	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
 	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
 	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
-	// Stat returns Unix stat(2) info
-	Stat(path string) (os.FileInfo, error)
+	// Size returns the size of the given file in bytes.
+	Size(path string) (size int64, err error)
 	// Open returns ReadSeekerCloser to access the file
 	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
@@ -45,6 +47,8 @@ type PackagePool interface {
 
 // LocalPackagePool is implemented by PackagePools residing on the same filesystem
 type LocalPackagePool interface {
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
 	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
 	GenerateTempPath(filename string) (string, error)
 	// Link generates hardlink to destination path
@@ -68,11 +72,19 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, baseName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
 // FileSystemPublishedStorage is published storage on filesystem
@@ -87,6 +99,36 @@ type PublishedStorageProvider interface {
 	GetPublishedStorage(name string) PublishedStorage
 }
 
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -98,7 +140,7 @@ type Progress interface {
 	// Flush returns when all queued messages are sent
 	Flush()
 	// InitBar starts progressbar for count bytes or count items
-	InitBar(count int64, isBytes bool)
+	InitBar(count int64, isBytes bool, barType BarType)
 	// ShutdownBar stops progress bar and hides it
 	ShutdownBar()
 	// AddBar increments progress for progress bar
@@ -116,13 +158,18 @@ type Progress interface {
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string) error
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
 }
 
+// ChecksumStorageProvider creates ChecksumStorage based on DB
+type ChecksumStorageProvider func(db database.ReaderWriter) ChecksumStorage
+
 // ChecksumStorage is stores checksums in some (persistent) storage
 type ChecksumStorage interface {
 	// Get finds checksums in DB by path

azure/azure.go

@@ -0,0 +1,136 @@
+package azure
+
+// Package azure handles publishing to Azure Storage
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/aptly-dev/aptly/aptly"
+)
+
+func isBlobNotFound(err error) bool {
+	var respErr *azcore.ResponseError
+	if errors.As(err, &respErr) {
+		return respErr.StatusCode == 404 // BlobNotFound
+	}
+	return false
+}
+
+type azContext struct {
+	client    *azblob.Client
+	container string
+	prefix    string
+}
+
+func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
+	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
+	}
+
+	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	result := &azContext{
+		client:    serviceClient,
+		container: container,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+func (az *azContext) blobPath(path string) string {
+	return filepath.Join(az.prefix, path)
+}
+
+func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(az.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	ctx := context.Background()
+	maxResults := int32(1)
+	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
+		Prefix:     &prefix,
+		MaxResults: &maxResults,
+		Include:    azblob.ListBlobsInclude{Metadata: true},
+	})
+
+	// Iterate over each page
+	for pager.More() {
+		page, err := pager.NextPage(ctx)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
+		}
+
+		for _, blob := range page.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, *blob.Name)
+			} else {
+				name := *blob.Name
+				paths = append(paths, name[len(prefix):])
+			}
+			b := *blob
+			md5 := b.Properties.ContentMD5
+			md5s = append(md5s, fmt.Sprintf("%x", md5))
+
+		}
+		if progress != nil {
+			time.Sleep(time.Duration(500) * time.Millisecond)
+			progress.AddBar(1)
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
+	uploadOptions := &azblob.UploadFileOptions{
+		BlockSize:   4 * 1024 * 1024,
+		Concurrency: 8,
+	}
+
+	path := az.blobPath(blobName)
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
+			BlobContentMD5: decodedMD5,
+		}
+	}
+
+	var err error
+	if file, ok := source.(*os.File); ok {
+		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
+	}
+
+	return err
+}
+
+// String
+func (az *azContext) String() string {
+	return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
+}

azure/azure_test.go

@@ -0,0 +1,12 @@
+package azure
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

azure/package_pool.go

@@ -0,0 +1,215 @@
+package azure
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+type PackagePool struct {
+	az *azContext
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates published storage from Azure storage credentials
+func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PackagePool{az: azctx}, nil
+}
+
+// String
+func (pool *PackagePool) String() string {
+	return pool.az.String()
+}
+
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
+	hash := checksums.SHA256
+	// Use the same path as the file pool, for compat reasons.
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
+}
+
+func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+	targetChecksums, err := checksumStorage.Get(poolPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		if err != nil {
+			if isBlobNotFound(err) {
+				return nil, nil
+			}
+
+			return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
+		}
+
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return targetChecksums, nil
+}
+
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	if progress != nil {
+		progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
+		defer progress.ShutdownBar()
+	}
+
+	paths, _, err := pool.az.internalFilelist("", progress)
+	return paths, err
+}
+
+func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
+	return "", errors.New("Azure package pool does not support legacy paths")
+}
+
+func (pool *PackagePool) Size(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	temp, err := os.CreateTemp("", "blob-download")
+	if err != nil {
+		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+	}
+	defer os.Remove(temp.Name())
+
+	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+	}
+
+	return temp, nil
+}
+
+func (pool *PackagePool) Remove(path string) (int64, error) {
+	serviceClient := pool.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(pool.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+
+	props, err := blobClient.GetProperties(context.TODO(), nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+	}
+
+	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	if err != nil {
+		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
+	}
+
+	return *props.ContentLength, nil
+}
+
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		var err error
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	path := pool.buildPoolPath(basename, checksums)
+	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
+	if err != nil {
+		return "", err
+	} else if targetChecksums != nil {
+		// target already exists
+		*checksums = *targetChecksums
+		return path, nil
+	}
+
+	source, err := os.Open(srcPath)
+	if err != nil {
+		return "", err
+	}
+	defer source.Close()
+
+	err = pool.az.putFile(path, source, checksums.MD5)
+	if err != nil {
+		return "", err
+	}
+
+	if !checksums.Complete() {
+		// need full checksums here
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	err = checksumStorage.Update(path, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	if poolPath == "" {
+		if checksums.SHA256 != "" {
+			poolPath = pool.buildPoolPath(basename, checksums)
+		} else {
+			// No checksums or pool path, so no idea what file to look for.
+			return "", false, nil
+		}
+	}
+
+	size, err := pool.Size(poolPath)
+	if err != nil {
+		return "", false, err
+	} else if size != checksums.Size {
+		return "", false, nil
+	}
+
+	targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
+	if err != nil {
+		return "", false, err
+	} else if targetChecksums == nil {
+		return "", false, nil
+	}
+
+	if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+		checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+		// wrong file?
+		return "", false, nil
+	}
+
+	// fill back checksums
+	*checksums = *targetChecksums
+	return poolPath, true, nil
+}

azure/package_pool_test.go

@@ -0,0 +1,257 @@
+package azure
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type PackagePoolSuite struct {
+	accountName, accountKey, endpoint string
+	pool, prefixedPool                *PackagePool
+	debFile                           string
+	cs                                aptly.ChecksumStorage
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
+	c.Assert(err, IsNil)
+
+	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = files.NewMockChecksumStorage()
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
+		"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
+	})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+
+	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	var checksum utils.ChecksumInfo
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	// import as different name
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+
+	// double import, should be ok
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum = utils.ChecksumInfo{}
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with path-relevant checksums already filled in
+	checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
+	path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	checksum := utils.ChecksumInfo{}
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with checksums missing (that aren't needed to find the path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	checksum = utils.ChecksumInfo{Size: checksum.Size}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
+		SHA256: "abc",
+		Size:   checksum.Size,
+	}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksums (that aren't needed to find the path)
+	// and no info in checksum storage
+	delete(s.cs.(*files.MockChecksumStorage).Store, path)
+	checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	checksum = utils.ChecksumInfo{Size: 13455}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestSize(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	size, err := s.pool.Size(path)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(2738))
+
+	_, err = s.pool.Size("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
+}

azure/public.go

@@ -0,0 +1,287 @@
+package azure
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pborman/uuid"
+	"github.com/pkg/errors"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Azure)
+type PublishedStorage struct {
+	prefix    string
+	az        *azContext
+	pathCache map[string]map[string]string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorage creates published storage from Azure storage credentials
+func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
+	azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	return &PublishedStorage{az: azctx}, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return storage.az.String()
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(_ string) error {
+	// no op for Azure
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+
+	sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename)
+	if err != nil {
+		return err
+	}
+
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.az.putFile(path, source, sourceMD5)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
+	path = storage.az.blobPath(path)
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range filelist {
+		blob := filepath.Join(path, filename)
+		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
+		if err != nil {
+			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
+		}
+	}
+
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	path = storage.az.blobPath(path)
+	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
+	}
+	return err
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedPrefix is desired prefix for the location in the pool.
+// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
+
+	relFilePath := filepath.Join(publishedRelPath, fileName)
+	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	poolPath := storage.az.blobPath(prefixRelFilePath)
+
+	if storage.pathCache == nil {
+		storage.pathCache = make(map[string]map[string]string)
+	}
+	pathCache := storage.pathCache[publishedPrefix]
+	if pathCache == nil {
+		paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
+		}
+
+		pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			pathCache[paths[i]] = md5s[i]
+		}
+		storage.pathCache[publishedPrefix] = pathCache
+	}
+
+	destinationMD5, exists := pathCache[relFilePath]
+	sourceMD5 := sourceChecksums.MD5
+
+	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+		}
+	}
+
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.az.putFile(relFilePath, source, sourceMD5)
+	if err == nil {
+		pathCache[relFilePath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
+	}
+
+	return err
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	paths, _, err := storage.az.internalFilelist(prefix, nil)
+	return paths, err
+}
+
+// Internal copy or move implementation
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
+	const leaseDuration = 30
+	leaseID := uuid.NewRandom().String()
+
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	srcBlobClient := containerClient.NewBlobClient(src)
+	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	}
+
+	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
+	if err != nil {
+		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	}
+	defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
+
+	dstBlobClient := containerClient.NewBlobClient(dst)
+	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
+		Metadata: metadata,
+	})
+
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	copyStatus := *copyResp.CopyStatus
+	for {
+		if copyStatus == blob.CopyStatusTypeSuccess {
+			if move {
+				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
+					AccessConditions: &blob.AccessConditions{
+						LeaseAccessConditions: &blob.LeaseAccessConditions{
+							LeaseID: &leaseID,
+						},
+					},
+				})
+				return err
+			}
+			return nil
+		} else if copyStatus == blob.CopyStatusTypePending {
+			time.Sleep(1 * time.Second)
+			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
+			if err != nil {
+				return fmt.Errorf("error getting copy progress %s", dst)
+			}
+			copyStatus = *getMetadata.CopyStatus
+
+			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
+			if err != nil {
+				return fmt.Errorf("error renewing source blob lease %s", src)
+			}
+		} else {
+			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
+		}
+	}
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */)
+}
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	metadata := make(map[string]*string)
+	metadata["SymLink"] = &src
+	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	_, err := blobClient.GetProperties(context.Background(), nil)
+	if err != nil {
+		if isBlobNotFound(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
+	}
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	serviceClient := storage.az.client.ServiceClient()
+	containerClient := serviceClient.NewContainerClient(storage.az.container)
+	blobClient := containerClient.NewBlobClient(path)
+	props, err := blobClient.GetProperties(context.Background(), nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to get blob properties: %v", err)
+	}
+
+	metadata := props.Metadata
+	if originalBlob, exists := metadata["original_blob"]; exists {
+		return *originalBlob, nil
+	}
+	return "", fmt.Errorf("error reading link %s: %v", path, err)
+}

azure/public_test.go

@@ -0,0 +1,374 @@
+package azure
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+        "bytes"
+
+        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type PublishedStorageSuite struct {
+	accountName, accountKey, endpoint string
+	storage, prefixedStorage          *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+const testContainerPrefix = "aptlytest-"
+
+func randContainer() string {
+	return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+	if n <= 0 {
+		panic("negative number")
+	}
+	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+	var bytes = make([]byte, n)
+	rand.Read(bytes)
+	for i, b := range bytes {
+		bytes[i] = alphanum[b%byte(len(alphanum))]
+	}
+	return string(bytes)
+}
+
+func (s *PublishedStorageSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+        publicAccessType := azblob.PublicAccessTypeContainer
+        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
+            Access: &publicAccessType,
+        })
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	c.Assert(err, IsNil)
+	data, err := ioutil.ReadAll(resp.Body)
+	c.Assert(err, IsNil)
+	return data
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+        serviceClient := s.storage.az.client.ServiceClient()
+        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
+        blobClient := containerClient.NewBlobClient(path)
+        _, err := blobClient.GetProperties(context.Background(), nil)
+	c.Assert(err, NotNil)
+
+        storageError, ok := err.(*azcore.ResponseError)
+	c.Assert(ok, Equals, true)
+	c.Assert(storageError.StatusCode, Equals, 404)
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	hash := md5.Sum(data)
+        uploadOptions := &azblob.UploadStreamOptions{
+            HTTPHeaders: &blob.HTTPHeaders{
+            	BlobContentMD5: hash[:],
+            },
+        }
+        reader := bytes.NewReader(data)
+        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+
+	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b+c.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+	s.AssertNoFile(c, "a/b c.txt")
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestFilelistPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a 1", "a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a b", "a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	err = s.prefixedStorage.Remove("xyz")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlus(c *C) {
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.RenameFile("source.txt", "dest.txt")
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!"))
+
+	exists, err := s.storage.FileExists("source.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
+
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
+
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
+	c.Assert(err, IsNil)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
+	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in azure test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

cmd/api_serve.go

@@ -1,15 +1,19 @@
 package cmd
 
 import (
+	stdcontext "context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
+	"os/signal"
+	"syscall"
 
-	"github.com/smira/aptly/api"
-	"github.com/smira/aptly/systemd/activation"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/api"
+	"github.com/aptly-dev/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}
@@ -55,6 +59,19 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listen := context.Flags().Lookup("listen").Value.String()
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
 
+	server := http.Server{Handler: api.Router(context)}
+
+	sigchan := make(chan os.Signal, 1)
+	signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
+	go (func() {
+		if _, ok := <-sigchan; ok {
+			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
+			context.TaskList().Wait()
+			server.Shutdown(stdcontext.Background())
+		}
+	})()
+	defer close(sigchan)
+
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
@@ -67,19 +84,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		}
 		defer listener.Close()
 
-		err = http.Serve(listener, api.Router(context))
-		if err != nil {
-			return fmt.Errorf("unable to serve: %s", err)
-		}
-		return nil
+		err = server.Serve(listener)
+	} else {
+		server.Addr = listen
+		err = server.ListenAndServe()
 	}
 
-	err = http.ListenAndServe(listen, api.Router(context))
-	if err != nil {
+	if err != nil && !errors.Is(err, http.ErrServerClosed) {
 		return fmt.Errorf("unable to serve: %s", err)
 	}
 
-	return err
+	return nil
 }
 
 func makeCmdAPIServe() *commander.Command {

cmd/cmd.go

@@ -8,8 +8,8 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -21,14 +21,14 @@ const (
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -118,8 +118,8 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
-	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\" for external gpg or \"internal\" for Go internal implementation)")
+	cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/config_show.go

@@ -5,19 +5,30 @@ import (
 	"fmt"
 
 	"github.com/smira/commander"
+	"gopkg.in/yaml.v3"
 )
 
-func aptlyConfigShow(cmd *commander.Command, args []string) error {
+func aptlyConfigShow(_ *commander.Command, _ []string) error {
+	showYaml := context.Flags().Lookup("yaml").Value.Get().(bool)
 
 	config := context.Config()
-	prettyJSON, err := json.MarshalIndent(config, "", "    ")
 
-	if err != nil {
-		return fmt.Errorf("unable to dump the config file: %s", err)
+	if showYaml {
+		yamlData, err := yaml.Marshal(&config)
+		if err != nil {
+			return fmt.Errorf("error marshaling to YAML: %s", err)
+		}
+
+		fmt.Println(string(yamlData))
+	} else {
+		prettyJSON, err := json.MarshalIndent(config, "", "    ")
+		if err != nil {
+			return fmt.Errorf("unable to dump the config file: %s", err)
+		}
+
+		fmt.Println(string(prettyJSON))
 	}
 
-	fmt.Println(string(prettyJSON))
-
 	return nil
 }
 
@@ -35,5 +46,6 @@ Example:
 
 `,
 	}
+	cmd.Flag.Bool("yaml", false, "show yaml config")
 	return cmd
 }

cmd/context.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	ctx "github.com/smira/aptly/context"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/flag"
 )
 
@@ -29,3 +29,8 @@ func InitContext(flags *flag.FlagSet) error {
 
 	return err
 }
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/db_cleanup.go

@@ -5,8 +5,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 )
 
@@ -21,6 +22,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
 	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
@@ -32,12 +34,12 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
 	}
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -59,15 +61,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	collectionFactory.Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
 	}
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -90,15 +94,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	collectionFactory.Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
 	}
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -118,17 +124,19 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	collectionFactory.Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
 	}
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
 		}
 		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 		if e != nil {
 			return e
 		}
@@ -150,9 +158,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	collectionFactory.Flush()
+
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
-	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
+	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
@@ -175,15 +185,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		if !dryRun {
-			db.StartBatch()
+			batch := db.CreateBatch()
 			err = toDelete.ForEach(func(ref []byte) error {
-				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+				return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("unable to delete by key: %s", err)
 			}
 
-			err = db.FinishBatch()
+			err = batch.Write()
 			if err != nil {
 				return fmt.Errorf("unable to write to DB: %s", err)
 			}
@@ -192,13 +202,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	collectionFactory.Flush()
+
 	// now, build a list of files that should be present in Repository (package pool)
 	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
 		if err2 != nil {
 			tail := ""
 			if verbose {
@@ -249,7 +261,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		if !dryRun {
-			context.Progress().InitBar(int64(len(filesToDelete)), false)
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
 
 			var size, totalSize int64
 			for _, file := range filesToDelete {

cmd/db_recover.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/database"
 	"github.com/smira/commander"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
@@ -15,7 +16,7 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = database.RecoverDB(context.DBPath())
+	err = goleveldb.RecoverDB(context.DBPath())
 
 	return err
 }

cmd/graph.go

@@ -4,16 +4,14 @@ import (
 	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 )
 
@@ -28,15 +26,15 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	layout := context.Flags().Lookup("layout").Value.String()
 
 	fmt.Printf("Generating graph...\n")
-	graph, err := deb.BuildGraph(context.CollectionFactory(), layout)
-
+	collectionFactory := context.NewCollectionFactory()
+	graph, err := deb.BuildGraph(collectionFactory, layout)
 	if err != nil {
 		return err
 	}
 
 	buf := bytes.NewBufferString(graph.String())
 
-	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	tempfile, err := os.CreateTemp("", "aptly-graph")
 	if err != nil {
 		return err
 	}
@@ -80,10 +78,6 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	defer func() {
-		_ = os.Remove(tempfilename)
-	}()
-
 	if output != "" {
 		err = utils.CopyFile(tempfilename, output)
 		if err != nil {
@@ -91,23 +85,16 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 		}
 
 		fmt.Printf("Output saved to %s\n", output)
+		_ = os.Remove(tempfilename)
 	} else {
 		command := getOpenCommand()
-		fmt.Printf("Rendered to %s file: %s, trying to open it with: %s %s...\n", format, tempfilename, command, tempfilename)
+		fmt.Printf("Displaying %s file: %s %s\n", format, command, tempfilename)
 
 		args := strings.Split(command, " ")
 
 		viewer := exec.Command(args[0], append(args[1:], tempfilename)...)
 		viewer.Stderr = os.Stderr
-		if err = viewer.Start(); err == nil {
-			// Wait for a second so that the visualizer has a chance to
-			// open the input file. This needs to be done even if we're
-			// waiting for the visualizer as it can be just a wrapper that
-			// spawns a browser tab and returns right away.
-			defer func(t <-chan time.Time) {
-				<-t
-			}(time.After(time.Second))
-		}
+		err = viewer.Start()
 	}
 
 	return err

cmd/mirror.go

@@ -3,24 +3,24 @@ package cmd
 import (
 	"strings"
 
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
-	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
-		return nil, nil
-	}
-
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring()
+	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -19,6 +19,11 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -36,12 +41,12 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs)
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
 
-	repo.Filter = context.Flags().Lookup("filter").Value.String()
+	repo.Filter = context.Flags().Lookup("filter").Value.String() // allows file/stdin with @
 	repo.FilterWithDeps = context.Flags().Lookup("filter-with-deps").Value.Get().(bool)
 	repo.SkipComponentCheck = context.Flags().Lookup("force-components").Value.Get().(bool)
 	repo.SkipArchitectureCheck = context.Flags().Lookup("force-architectures").Value.Get().(bool)
@@ -58,12 +63,13 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -94,12 +100,14 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -15,8 +15,9 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -28,7 +29,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -40,7 +41,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
+	err = collectionFactory.RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/mirror_edit.go

@@ -3,7 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -15,7 +16,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -25,16 +27,25 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
-			repo.Filter = flag.Value.String()
+			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
+		case "ignore-signatures":
+			ignoreSignatures = true
 		}
 	})
 
@@ -51,14 +62,23 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
 
-		err = repo.Fetch(context.Downloader(), nil)
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -83,10 +103,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
-	cmd.Flag.String("filter", "", "filter packages in mirror")
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
+	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_list.go

@@ -1,25 +1,38 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	if jsonFlag {
+		return aptlyMirrorListJSON(cmd, args)
+	}
+
+	return aptlyMirrorListTxt(cmd, args)
+}
+
+func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -52,6 +65,32 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdMirrorList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorList,
@@ -66,6 +105,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/mirror_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -20,7 +20,8 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
@@ -30,13 +31,13 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	_, err = collectionFactory.RemoteRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/mirror_show.go

@@ -1,30 +1,44 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorShowJSON(cmd, args)
+	}
+
+	return aptlyMirrorShowTxt(cmd, args)
+}
+
+func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -72,13 +86,56 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList())
+			ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
 	return err
 }
 
+func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				repo.Packages = append(repo.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(repo.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdMirrorShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorShow,
@@ -94,6 +151,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
 
 	return cmd

cmd/mirror_update.go

@@ -3,14 +3,13 @@ package cmd
 import (
 	"fmt"
 	"os"
-	"os/signal"
 	"strings"
 	"sync"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -24,12 +23,13 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -42,21 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
-	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
+	ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
 		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
 	}
 
-	err = repo.Fetch(context.Downloader(), verifier)
+	err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -86,8 +89,8 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(),
-		context.CollectionFactory().ChecksumCollection(), skipExistingPackages)
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -98,12 +101,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+			collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
 	repo.MarkAsUpdating()
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -113,23 +116,13 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	// Catch ^C
-	sigch := make(chan os.Signal)
-	signal.Notify(sigch, os.Interrupt)
-	defer signal.Stop(sigch)
-
-	abort := make(chan struct{})
-	go func() {
-		<-sigch
-		signal.Stop(sigch)
-		close(abort)
-	}()
+	context.GoContextHandleSignals()
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
 
 	// Download from the queue
-	context.Progress().InitBar(downloadSize, true)
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
 
 	downloadQueue := make(chan int)
 
@@ -148,7 +141,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		for idx := range queue {
 			select {
 			case downloadQueue <- idx:
-			case <-abort:
+			case <-context.Done():
 				return
 			}
 		}
@@ -173,7 +166,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 					var e error
 
 					// provision download location
-					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
+						task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
+					} else {
+						var file *os.File
+						file, e = os.CreateTemp("", task.File.Filename)
+						if e == nil {
+							task.TempDownPath = file.Name()
+							file.Close()
+						}
+					}
 					if e != nil {
 						pushError(e)
 						continue
@@ -181,53 +183,61 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 					// download file...
 					e = context.Downloader().DownloadWithChecksum(
+						context,
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
-						ignoreMismatch,
-						maxTries)
+						ignoreChecksums)
 					if e != nil {
 						pushError(e)
 						continue
 					}
-				case <-abort:
+
+					task.Done = true
+				case <-context.Done():
 					return
 				}
 			}
 		}()
 	}
 
-	// Wait for all downloads to finish
+	// Wait for all download goroutines to finish
 	wg.Wait()
 
-	select {
-	case <-abort:
-		return fmt.Errorf("unable to update: interrupted")
-	default:
-	}
-
 	context.Progress().ShutdownBar()
 
-	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
-	}
-
 	err = context.ReOpenDatabase()
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	defer func() {
+		for _, task := range queue {
+			if task.TempDownPath == "" {
+				continue
+			}
+
+			if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
+				fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
+			}
+		}
+	}()
+
 	// Import downloaded files
-	context.Progress().InitBar(int64(len(queue)), false)
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 
 	for idx := range queue {
-
 		context.Progress().AddBar(1)
 
 		task := &queue[idx]
 
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
 		// and import it back to the pool
-		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
 		if err != nil {
 			return fmt.Errorf("unable to import file: %s", err)
 		}
@@ -237,23 +247,27 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 			additionalTask.File.PoolPath = task.File.PoolPath
 			additionalTask.File.Checksums = task.File.Checksums
 		}
-
-		select {
-		case <-abort:
-			return fmt.Errorf("unable to update: interrupted")
-		default:
-		}
 	}
 
 	context.Progress().ShutdownBar()
 
-	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(collectionFactory, context.Progress())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	context.Progress().Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	context.Progress().Printf("\nMirror `%s` has been updated successfully.\n", repo.Name)
 	return err
 }
 
@@ -279,6 +293,7 @@ Example:
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 

cmd/package_search.go

@@ -3,8 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -21,7 +21,11 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	if len(args) == 1 {
-		q, err = query.Parse(args[0])
+		value, err := GetStringOrFileContent(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -29,7 +33,8 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 		q = &deb.MatchAllQuery{}
 	}
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 	if result.Len() == 0 {
 		return fmt.Errorf("no results")
 	}
@@ -48,6 +53,7 @@ func makeCmdPackageSearch() *commander.Command {
 		Long: `
 Command search displays list of packages in whole DB that match package query.
 
+Use '@file' to read query from file or '@-' for stdin.
 If query is not specified, all the packages are displayed.
 
 Example:

cmd/package_show.go

@@ -5,16 +5,16 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func printReferencesTo(p *deb.Package) (err error) {
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -29,8 +29,8 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -45,8 +45,8 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -66,7 +66,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	q, err := query.Parse(args[0])
+	value, err := GetStringOrFileContent(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to read package query from file %s: %w", args[0], err)
+	}
+	q, err := query.Parse(value)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -76,10 +80,11 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 	w := bufio.NewWriter(os.Stdout)
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false)
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
 		w.Flush()
 		fmt.Printf("\n")
 
@@ -104,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p)
+			printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 
@@ -129,6 +134,8 @@ matching query. Information from Debian control file is displayed.
 Optionally information about package files and
 inclusion into mirrors/snapshots/local repos is shown.
 
+Use '@file' to read query from file or '@-' for stdin.
+
 Example:
 
     $ aptly package show 'nginx-light_1.2.1-2.2+wheezy2_i386'

cmd/publish.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -34,10 +34,26 @@ func makeCmdPublish() *commander.Command {
 			makeCmdPublishDrop(),
 			makeCmdPublishList(),
 			makeCmdPublishRepo(),
+			makeCmdPublishShow(),
 			makeCmdPublishSnapshot(),
+			makeCmdPublishSource(),
 			makeCmdPublishSwitch(),
 			makeCmdPublishUpdate(),
-			makeCmdPublishShow(),
+		},
+	}
+}
+
+func makeCmdPublishSource() *commander.Command {
+	return &commander.Command{
+		UsageLine: "source",
+		Short:     "manage sources of published repository",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSourceAdd(),
+			makeCmdPublishSourceDrop(),
+			makeCmdPublishSourceList(),
+			makeCmdPublishSourceRemove(),
+			makeCmdPublishSourceReplace(),
+			makeCmdPublishSourceUpdate(),
 		},
 	}
 }

cmd/publish_drop.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -23,8 +23,11 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 
 	storage, prefix := deb.ParsePrefix(param)
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		collectionFactory, context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -50,6 +53,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_list.go

@@ -1,27 +1,43 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishListJSON(cmd, args)
+	}
+
+	return aptlyPublishListTxt(cmd, args)
+}
+
+func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
 		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
 			return e
 		}
 
@@ -61,6 +77,42 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		repos = append(repos, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GetPath() < repos[j].GetPath()
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdPublishList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishList,
@@ -75,6 +127,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/publish_repo.go

@@ -37,16 +37,21 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_show.go

@@ -1,20 +1,32 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
 func aptlyPublishShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishShowJSON(cmd, args)
+	}
+
+	return aptlyPublishShowTxt(cmd, args)
+}
+
+func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
 	distribution := args[0]
 	param := "."
 
@@ -24,7 +36,8 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 
 	storage, prefix := deb.ParsePrefix(param)
 
-	repo, err := context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -39,16 +52,17 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, " "))
 
 	fmt.Printf("Sources:\n")
-	for component, sourceID := range repo.Sources {
+	for _, component := range repo.Components() {
+		sourceID := repo.Sources[component]
 		var name string
 		if repo.SourceKind == deb.SourceSnapshot {
-			source, e := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
 			if e != nil {
 				continue
 			}
 			name = source.Name
 		} else if repo.SourceKind == deb.SourceLocalRepo {
-			source, e := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			source, e := collectionFactory.LocalRepoCollection().ByUUID(sourceID)
 			if e != nil {
 				continue
 			}
@@ -63,6 +77,36 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdPublishShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishShow,
@@ -77,5 +121,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+
 	return cmd
 }

cmd/publish_snapshot.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -15,6 +15,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	collectionFactory := context.NewCollectionFactory()
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -43,12 +44,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -79,12 +80,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			localRepo, err = collectionFactory.LocalRepoCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -115,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	origin := context.Flags().Lookup("origin").Value.String()
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
+	multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -130,6 +132,8 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.ButAutomaticUpgrades = butAutomaticUpgrades
 	}
 	published.Label = context.Flags().Lookup("label").Value.String()
+	published.Suite = context.Flags().Lookup("suite").Value.String()
+	published.Codename = context.Flags().Lookup("codename").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
 
@@ -137,9 +141,22 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
+	published.SkipBz2 = context.Config().SkipBz2Publishing
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -150,16 +167,15 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
 	if forceOverwrite {
-		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
-			"the same package pool.\n")
+		context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -217,16 +233,21 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "overwrite origin name to publish")
 	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
+	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_source_add.go

@@ -0,0 +1,94 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceAdd(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if exists {
+			return fmt.Errorf("unable to add: component '%s' has already been added", component)
+		}
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceAdd,
+		UsageLine: "add <distribution> <source>",
+		Short:     "add source components to a published repo",
+		Long: `
+The command adds components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source add -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source add -component=contrib wheezy ppa wheezy-contrib
+
+This command assigns the snapshot wheezy-contrib to the component contrib and
+adds it to published repository revision of ppa/wheezy.
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_drop.go

@@ -0,0 +1,62 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceDrop(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	published.DropRevision()
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("Source changes have been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishSourceDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceDrop,
+		UsageLine: "drop <distribution>",
+		Short:     "drop pending source component changes of a published repository",
+		Long: `
+Remove all pending changes what would be applied with a subsequent 'aptly publish update'.
+
+Example:
+
+    $ aptly publish source drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-drop", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_list.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceList(cmd *commander.Command, args []string) error {
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	distribution := args[0]
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	published, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(published, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	if published.Revision == nil {
+		return fmt.Errorf("unable to list: no source changes exist")
+	}
+
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishSourceListJSON(published)
+	}
+
+	return aptlyPublishSourceListTxt(published)
+}
+
+func aptlyPublishSourceListTxt(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	fmt.Printf("Sources:\n")
+	for _, component := range revision.Components() {
+		name := revision.Sources[component]
+		fmt.Printf("  %s: %s [%s]\n", component, name, published.SourceKind)
+	}
+
+	return nil
+}
+
+func aptlyPublishSourceListJSON(published *deb.PublishedRepo) error {
+	revision := published.Revision
+
+	output, err := json.MarshalIndent(revision.SourceList(), "", "  ")
+	if err != nil {
+		return fmt.Errorf("unable to list: %s", err)
+	}
+
+	fmt.Println(string(output))
+
+	return nil
+}
+
+func makeCmdPublishSourceList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceList,
+		UsageLine: "list <distribution>",
+		Short:     "lists revision of published repository",
+		Long: `
+Command lists sources of a published repository.
+
+Example:
+
+    $ aptly publish source list wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-list", flag.ExitOnError),
+	}
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_remove.go

@@ -0,0 +1,86 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceRemove(cmd *commander.Command, args []string) error {
+	if len(args) < 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(components) == 0 {
+		return fmt.Errorf("unable to remove: missing components, specify at least one component")
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for _, component := range components {
+		name, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to remove: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Removing component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		delete(sources, component)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceRemove,
+		UsageLine: "remove <distribution> [[<endpoint>:]<prefix>] <source>",
+		Short:     "remove source components from a published repo",
+		Long: `
+The command removes source components (snapshot / local repo) from a published repository.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be removed, e.g.:
+
+Example:
+
+	$ aptly publish source remove -component=contrib,non-free wheezy filesystem:symlink:debian
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-remove", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to remove (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_replace.go

@@ -0,0 +1,89 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceReplace(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+	context.Progress().Printf("Replacing source list...\n")
+	clear(sources)
+
+	for i, component := range components {
+		name := names[i]
+		context.Progress().Printf("Adding component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceReplace() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceReplace,
+		UsageLine: "replace <distribution> <source>",
+		Short:     "replace the source components of a published repository",
+		Long: `
+The command replaces the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source replace -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source replace -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-add", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_source_update.go

@@ -0,0 +1,91 @@
+package cmd
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPublishSourceUpdate(cmd *commander.Command, args []string) error {
+	if len(args) < 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	distribution := args[0]
+	names := args[1:]
+	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+
+	if len(names) != len(components) {
+		return fmt.Errorf("mismatch in number of components (%d) and sources (%d)", len(components), len(names))
+	}
+
+	prefix := context.Flags().Lookup("prefix").Value.String()
+	storage, prefix := deb.ParsePrefix(prefix)
+
+	collectionFactory := context.NewCollectionFactory()
+	published, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	revision := published.ObtainRevision()
+	sources := revision.Sources
+
+	for i, component := range components {
+		name := names[i]
+		_, exists := sources[component]
+		if !exists {
+			return fmt.Errorf("unable to update: component '%s' does not exist", component)
+		}
+		context.Progress().Printf("Updating component '%s' with source '%s' [%s]...\n", component, name, published.SourceKind)
+
+		sources[component] = name
+	}
+
+	err = collectionFactory.PublishedRepoCollection().Update(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	context.Progress().Printf("\nYou can run 'aptly publish update %s %s' to update the content of the published repository.\n",
+		distribution, published.StoragePrefix())
+
+	return err
+}
+
+func makeCmdPublishSourceUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSourceUpdate,
+		UsageLine: "update <distribution> <source>",
+		Short:     "update the source components of a published repository",
+		Long: `
+The command updates the source components of a snapshot or local repository to be published.
+
+This does not publish the changes directly, but rather schedules them for a subsequent 'aptly publish update'.
+
+The flag -component is mandatory. Use a comma-separated list of components, if
+multiple components should be modified. The number of given components must be
+equal to the number of given sources, e.g.:
+
+	aptly publish source update -component=main,contrib wheezy wheezy-main wheezy-contrib
+
+Example:
+
+	$ aptly publish source update -component=contrib wheezy ppa wheezy-contrib
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-source-update", flag.ExitOnError),
+	}
+	cmd.Flag.String("prefix", ".", "publishing prefix in the form of [<endpoint>:]<prefix>")
+	cmd.Flag.String("component", "", "component names to add (for multi-component publishing, separate components with commas)")
+
+	return cmd
+}

cmd/publish_switch.go

@@ -4,14 +4,17 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
-	var err error
+	var (
+		err   error
+		names []string
+	)
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
 
@@ -23,11 +26,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 	distribution := args[0]
 	param := "."
 
-	var (
-		names    []string
-		snapshot *deb.Snapshot
-	)
-
 	if len(args) == len(components)+2 {
 		param = args[1]
 		names = args[2:]
@@ -39,18 +37,19 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	if published.SourceKind != deb.SourceSnapshot {
-		return fmt.Errorf("unable to update: not a snapshot publish")
+		return fmt.Errorf("unable to switch: not a published snapshot repository")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+		return fmt.Errorf("unable to switch: %s", err)
 	}
 
 	publishedComponents := published.Components()
@@ -62,17 +61,18 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("mismatch in number of components (%d) and snapshots (%d)", len(components), len(names))
 	}
 
+	snapshotCollection := collectionFactory.SnapshotCollection()
 	for i, component := range components {
 		if !utils.StrSliceHasItem(publishedComponents, component) {
-			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
+			return fmt.Errorf("unable to switch: component %s does not exist in published repository", component)
 		}
 
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		snapshot, err := snapshotCollection.ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -95,23 +95,33 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, components, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to switch: %s", err)
+		}
 	}
 
-	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been successfully switched to new source.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -119,15 +129,15 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 func makeCmdPublishSwitch() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishSwitch,
-		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-snapshot>",
-		Short:     "update published repository by switching to new snapshot",
+		UsageLine: "switch <distribution> [[<endpoint>:]<prefix>] <new-source>",
+		Short:     "update published repository by switching to new source",
 		Long: `
-Command switches in-place published snapshots with new snapshot contents. All
+Command switches in-place published snapshots with new source contents. All
 publishing parameters are preserved (architecture list, distribution,
 component).
 
 For multiple component repositories, flag -component should be given with
-list of components to update. Corresponding snapshots should be given in the
+list of components to update. Corresponding sources should be given in the
 same order, e.g.:
 
 	aptly publish switch -component=main,contrib wheezy wh-main wh-contrib
@@ -144,13 +154,16 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/publish_update.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -25,23 +25,20 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != deb.SourceLocalRepo {
-		return fmt.Errorf("unable to update: not a local repository publish")
-	}
-
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	components := published.Components()
-	for _, component := range components {
-		published.UpdateLocalRepo(component)
+	result, err := published.Update(collectionFactory, context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
 	}
 
 	signer, err := getSigner(context.Flags())
@@ -59,23 +56,35 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	if context.Flags().IsSet("multi-dist") {
+		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
+	if !skipCleanup {
+		cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
+		cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
-	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
+	context.Progress().Printf("\nPublished %s repository %s has been updated successfully.\n", published.SourceKind, published.String())
 
 	return err
 }
@@ -84,15 +93,21 @@ func makeCmdPublishUpdate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishUpdate,
 		UsageLine: "update <distribution> [[<endpoint>:]<prefix>]",
-		Short:     "update published local repository",
+		Short:     "update published repository",
 		Long: `
-Command re-publishes (updates) published local repository. <distribution>
-and <prefix> should be occupied with local repository published
-using command aptly publish repo. Update happens in-place with
-minimum possible downtime for published repository.
+The command updates updates a published repository after applying pending changes to the sources.
 
-For multiple component published repositories, all local repositories
-are updated.
+For published local repositories:
+
+    * update to match local repository contents
+
+For published snapshots:
+
+    * switch components to new snapshot
+
+The update happens in-place with minimum possible downtime for published repository.
+
+For multiple component published repositories, all local repositories are updated.
 
 Example:
 
@@ -103,12 +118,15 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
+	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd
 }

cmd/repo_add.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -22,42 +22,45 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	verifier := context.GetVerifier()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	var packageFiles, failedFiles []string
+	var packageFiles, otherFiles, failedFiles []string
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
-		context.CollectionFactory().ChecksumCollection())
+		collectionFactory.PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		collectionFactory.ChecksumCollection)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}
@@ -88,7 +91,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 func makeCmdRepoAdd() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoAdd,
-		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		UsageLine: "add <name> (<package file.deb>|<directory>)...",
 		Short:     "add packages to local repository",
 		Long: `
 Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.

cmd/repo_create.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -27,15 +27,16 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	collectionFactory := context.NewCollectionFactory()
 	if len(args) == 4 {
 		var snapshot *deb.Snapshot
 
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(args[3])
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(args[3])
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
@@ -43,7 +44,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		repo.UpdateRefList(snapshot.RefList())
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}

cmd/repo_drop.go

@@ -15,17 +15,18 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	published := collectionFactory.PublishedRepoCollection().ByLocalRepo(repo)
 	if len(published) > 0 {
 		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -37,7 +38,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -49,7 +50,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
+	err = collectionFactory.LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/repo_edit.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/AlekSi/pointer"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -16,12 +16,13 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -52,7 +53,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/repo_include.go

@@ -1,16 +1,12 @@
 package cmd
 
 import (
-	"bytes"
 	"fmt"
-	"os"
-	"path/filepath"
 	"text/template"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -33,10 +29,16 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
-	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	ignoreSignatures := context.Config().GpgDisableVerify
+	if context.Flags().IsSet("ignore-signatures") {
+		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
+	}
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+	collectionFactory := context.NewCollectionFactory()
 
-	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
 	if err != nil {
 		return fmt.Errorf("error parsing -repo template: %s", err)
 	}
@@ -59,139 +61,15 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
 
-	var changesFiles, failedFiles, processedFiles []string
+	var changesFiles, failedFiles, failedFiles2 []string
 
 	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
-
-	for _, path := range changesFiles {
-		var changes *deb.Changes
-
-		changes, err = deb.NewChanges(path)
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", path, err)
-			continue
-		}
-
-		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		err = changes.Prepare()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		repoName := &bytes.Buffer{}
-		err = repoTemplate.Execute(repoName, changes.Stanza)
-		if err != nil {
-			return fmt.Errorf("error applying template to repo: %s", err)
-		}
-
-		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
-
-		var repo *deb.LocalRepo
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		currentUploaders := uploaders
-		if repo.Uploaders != nil {
-			currentUploaders = repo.Uploaders
-			for i := range currentUploaders.Rules {
-				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
-				if err != nil {
-					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
-				}
-			}
-		}
-
-		if currentUploaders != nil {
-			if err = currentUploaders.IsAllowed(changes); err != nil {
-				failedFiles = append(failedFiles, path)
-				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
-					changes.ChangesName, changes.SignatureKeys, err)
-				changes.Cleanup()
-				continue
-			}
-		}
-
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return fmt.Errorf("unable to load repo: %s", err)
-		}
-
-		var list *deb.PackageList
-		list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-		if err != nil {
-			return fmt.Errorf("unable to load packages: %s", err)
-		}
-
-		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
-
-		var restriction deb.PackageQuery
-
-		restriction, err = changes.PackageQuery()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		var processedFiles2, failedFiles2 []string
-
-		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())
-
-		if err != nil {
-			return fmt.Errorf("unable to import package files: %s", err)
-		}
-
-		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-		if err != nil {
-			return fmt.Errorf("unable to save: %s", err)
-		}
-
-		err = changes.Cleanup()
-		if err != nil {
-			return err
-		}
-
-		for _, file := range failedFiles2 {
-			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		for _, file := range processedFiles2 {
-			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		processedFiles = append(processedFiles, path)
-	}
-
-	if !noRemoveFiles {
-		processedFiles = utils.StrSliceDeduplicate(processedFiles)
-
-		for _, file := range processedFiles {
-			err = os.Remove(file)
-			if err != nil {
-				return fmt.Errorf("unable to remove file: %s", err)
-			}
-		}
-	}
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
+		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+		context.PackagePool(), collectionFactory.ChecksumCollection,
+		uploaders, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
 
 	if len(failedFiles) > 0 {
 		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")
@@ -208,14 +86,14 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 func makeCmdRepoInclude() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoInclude,
-		UsageLine: "include <file.changes>|<directory> ...",
+		UsageLine: "include (<file.changes>|<directory>)...",
 		Short:     "add packages to local repositories based on .changes files",
 		Long: `
 Command include looks for .changes files in list of arguments or specified directories. Each
 .changes file is verified, parsed, referenced files are put into separate temporary directory
 and added into local repository. Successfully imported files are removed by default.
 
-Additionally uploads could be restricted with <uploaders.json> file. Rules in this file control
+Additionally uploads could be restricted with 'uploaders.json' file. Rules in this file control
 uploads based on GPG key ID of .changes file signature and queries on .changes file fields.
 
 Example:

cmd/repo_list.go

@@ -1,29 +1,42 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoListJSON(cmd, args)
+	}
+
+	return aptlyRepoListTxt(cmd, args)
+}
+
+func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 			if e != nil {
 				return e
 			}
@@ -58,6 +71,36 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
+	var err error
+
+	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdRepoList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoList,
@@ -72,6 +115,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/repo_move.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -19,12 +19,13 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	command := cmd.Name()
 
-	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -35,7 +36,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	)
 
 	if command == "copy" || command == "move" { // nolint: goconst
-		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+		srcRepo, err = collectionFactory.LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -44,7 +45,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -53,12 +54,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+		srcRemoteRepo, err = collectionFactory.RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -74,12 +75,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -109,13 +110,24 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i := 0; i < len(args)-2; i++ {
-		queries[i], err = query.Parse(args[i+2])
+		value, err := GetStringOrFileContent(args[i+2])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+2], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 	}
 
-	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
+	toProcess, err := srcList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            dstList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -151,7 +163,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -159,7 +171,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}
@@ -178,6 +190,8 @@ func makeCmdRepoMove() *commander.Command {
 Command move moves packages matching <package-query> from local repo
 <src-name> to local repo <dst-name>.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo move testing stable 'myapp (=0.1.12)'

cmd/repo_remove.go

@@ -3,8 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -18,33 +18,38 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	queries := make([]deb.PackageQuery, len(args)-1)
 	for i := 0; i < len(args)-1; i++ {
-		queries[i], err = query.Parse(args[i+1])
+		value, err := GetStringOrFileContent(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[i+1], err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to remove: %s", err)
 		}
 	}
 
 	list.PrepareIndex()
-	toRemove, err := list.Filter(queries, false, nil, 0, nil)
+	toRemove, err := list.Filter(deb.FilterOptions{Queries: queries})
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -60,7 +65,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -80,6 +85,8 @@ Commands removes packages matching <package-query> from local repository
 snapshots, they can be removed completely (including files) by running
 'aptly db cleanup'.
 
+Use '@file' to read package queries from file or '@-' for stdin.
+
 Example:
 
   $ aptly repo remove testing 'myapp (=0.1.12)'

cmd/repo_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -20,18 +20,19 @@ func aptlyRepoRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.LocalRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	_, err = collectionFactory.LocalRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/repo_show.go

@@ -1,27 +1,42 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoShowJSON(cmd, args)
+	}
+
+	return aptlyRepoShowTxt(cmd, args)
+}
+
+func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -37,7 +52,49 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList())
+		ListPackagesRefList(repo.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	packageList := []string{}
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err == nil {
+				packageList = list.FullNames()
+			}
+		}
+
+		sort.Strings(packageList)
+	}
+
+	// merge the repo object with the package list
+	var output []byte
+	if output, err = json.MarshalIndent(struct {
+		*deb.LocalRepo
+		Packages []string
+	}{repo, packageList}, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -57,6 +114,7 @@ ex:
 		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/run.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	ctx "github.com/smira/aptly/context"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/commander"
 )
 

cmd/serve.go

@@ -8,9 +8,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -29,12 +29,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	// anything else must fail.
 	// E.g.: Running the service under a different user may lead to a rootDir
 	// that exists but is not usable due to access permissions.
-	err = utils.DirIsAccessible(context.Config().RootDir)
+	err = utils.DirIsAccessible(context.Config().GetRootDir())
 	if err != nil {
 		return err
 	}
 
-	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
+	collectionFactory := context.NewCollectionFactory()
+	if collectionFactory.PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
@@ -56,11 +57,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
+	sources := make(sort.StringSlice, 0, collectionFactory.PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 		if e != nil {
 			return e
 		}

cmd/snapshot_create.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -13,13 +13,14 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
+	collectionFactory := context.NewCollectionFactory()
 	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
 		repoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -29,7 +30,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -44,12 +45,12 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 
 		localRepoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -70,7 +71,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}
@@ -83,7 +84,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 func makeCmdSnapshotCreate() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotCreate,
-		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		UsageLine: "create <name> (from mirror <mirror-name> | from repo <repo-name> | empty)",
 		Short:     "creates snapshot of mirror (local repository) contents",
 		Long: `
 Command create <name> from mirror makes persistent immutable snapshot of remote

cmd/snapshot_diff.go

@@ -15,31 +15,32 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	}
 
 	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name-a> snapshot
-	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshotA, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	snapshotB, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_drop.go

@@ -15,18 +15,19 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
+	published := collectionFactory.PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -38,7 +39,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
+		snapshots := collectionFactory.SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -49,7 +50,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
+	err = collectionFactory.SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/snapshot_filter.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -19,21 +19,22 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	}
 
 	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <source> snapshot
-	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	source, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
 	// Convert snapshot to package list
 	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
-	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -59,14 +60,25 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-2)
 	for i, arg := range args[2:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := packageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  withDeps,
+		Source:            nil,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
@@ -75,7 +87,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
 		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
 
-	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -95,9 +107,11 @@ Command filter does filtering in snapshot <source>, producing another
 snapshot <destination>. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
-    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'
 `,
 		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
 	}

cmd/snapshot_list.go

@@ -1,23 +1,36 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlySnapshotListJSON(cmd, args)
+	}
+
+	return aptlySnapshotListTxt(cmd, args)
+}
+
+func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	collection := context.CollectionFactory().SnapshotCollection()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
 		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
@@ -46,6 +59,29 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
+	var err error
+
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	collection := context.NewCollectionFactory().SnapshotCollection()
+
+	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
+	i := 0
+	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		jsonSnapshots[i] = snapshot
+		i++
+		return nil
+	})
+	if output, e := json.MarshalIndent(jsonSnapshots, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdSnapshotList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotList,
@@ -60,6 +96,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
 

cmd/snapshot_merge.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -15,15 +15,16 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	collectionFactory := context.NewCollectionFactory()
 	sources := make([]*deb.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
+		sources[i], err = collectionFactory.SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
@@ -56,7 +57,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_pull.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -21,25 +21,26 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
 	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name> snapshot
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	source, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -49,12 +50,12 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	// Convert snapshot to package list
 	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -87,7 +88,11 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	// Initial queries out of arguments
 	queries := make([]deb.PackageQuery, len(args)-3)
 	for i, arg := range args[3:] {
-		queries[i], err = query.Parse(arg)
+		value, err := GetStringOrFileContent(arg)
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", arg, err)
+		}
+		queries[i], err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to parse query: %s", err)
 		}
@@ -96,7 +101,14 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := sourcePackageList.Filter(deb.FilterOptions{
+		Queries:           queries,
+		WithDependencies:  !noDeps,
+		Source:            packageList,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -111,7 +123,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		// If we haven't seen such name-architecture pair and were instructed to remove, remove it
 		if !noRemove && !seen {
 			// Remove all packages with the same name and architecture
-			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
+			pS := packageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true, false)
 			for _, p := range pS {
 				packageList.Remove(p)
 				context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
@@ -137,7 +149,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = context.CollectionFactory().SnapshotCollection().Add(destination)
+		err = collectionFactory.SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -159,6 +171,8 @@ versions from <source> following dependencies. New snapshot <destination>
 is created as a result of this process. Packages could be specified simply
 as 'package-name' or as package queries.
 
+Use '@file' syntax to read package queries from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server

cmd/snapshot_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -19,19 +19,20 @@ func aptlySnapshotRename(cmd *commander.Command, args []string) error {
 	}
 
 	oldName, newName := args[0], args[1]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	snapshot, err = collectionFactory.SnapshotCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	_, err = collectionFactory.SnapshotCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
 	}
 
 	snapshot.Name = newName
-	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/snapshot_search.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -23,17 +23,18 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	name := args[0]
 	command := cmd.Parent.Name()
+	collectionFactory := context.NewCollectionFactory()
 
 	var reflist *deb.PackageRefList
 
 	if command == "snapshot" { // nolint: goconst
 		var snapshot *deb.Snapshot
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -41,12 +42,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		reflist = snapshot.RefList()
 	} else if command == "mirror" {
 		var repo *deb.RemoteRepo
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -54,12 +55,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		reflist = repo.RefList()
 	} else if command == "repo" { // nolint: goconst
 		var repo *deb.LocalRepo
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -69,7 +70,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		panic("unknown command")
 	}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -77,7 +78,11 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	list.PrepareIndex()
 
 	if len(args) == 2 {
-		q, err = query.Parse(args[1])
+		value, err := GetStringOrFileContent(args[1])
+		if err != nil {
+			return fmt.Errorf("unable to read package query from file %s: %w", args[1], err)
+		}
+		q, err = query.Parse(value)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -102,8 +107,13 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 	}
 
-	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList, context.Progress())
+	result, err := list.Filter(deb.FilterOptions{
+		Queries:           []deb.PackageQuery{q},
+		WithDependencies:  withDeps,
+		DependencyOptions: context.DependencyOptions(),
+		Architectures:     architecturesList,
+		Progress:          context.Progress(),
+	})
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -128,6 +138,8 @@ Command search displays list of packages in snapshot that match package query
 
 If query is not specified, all the packages are displayed.
 
+Use '@file' syntax to read package query from file and '@-' to read from stdin.
+
 Example:
 
     $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'