New upstream version 1.5.0+ds1

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+max-line-length = 200
+ignore = E126,E241,E741,W504
+include =
+    system
+exclude =
+    system/env

.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
+
+name: CI
+
+on:
+  pull_request:
+  push:
+    tags:
+      - 'v*'
+    branches:
+      - 'master'
+
+defaults:
+  run:
+    # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
+    shell: bash --noprofile --norc -eo pipefail -x {0}
+
+env:
+  DEBIAN_FRONTEND: noninteractive
+
+jobs:
+  build:
+    name: test
+    runs-on: ubuntu-20.04
+    continue-on-error: ${{ matrix.allow_failure }}
+    timeout-minutes: 30
+    strategy:
+      fail-fast: true
+      matrix:
+        go: [1.16, 1.17, 1.18]
+        run_long_tests: [no]
+        allow_failure: [false]
+        include:
+          # Disable this for now as it's not clear how to select the latest master branch
+          # version of go using actions/setup-go@v2.
+          # - go: master
+          #   run_long_tests: no
+          #   allow_failure: true
+          - go: 1.17
+            run_long_tests: yes
+            allow_failure: false
+
+    env:
+      NO_FTP_ACCESS: yes
+      BOTO_CONFIG: /dev/null
+      GO111MODULE: "on"
+      GOPROXY: "https://proxy.golang.org"
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: v1.45.0
+
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install O/S packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
+
+      - name: Install Python packages
+        run: |
+          pip install six packaging appdirs virtualenv
+          pip install -U pip setuptools
+          pip install -r system/requirements.txt
+
+      - name: Install Azurite
+        id: azuright
+        uses: potatoqualitee/azuright@v1.1
+        with:
+          directory: ${{ runner.temp }}
+
+      - name: Get aptly version
+        run: |
+           make version
+
+      - name: Make
+        env:
+          RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
+          AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+        run: |
+          make
+
+      - name: Upload code coverage
+        if: matrix.run_long_tests
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage.txt
+
+  release:
+    name: release
+    needs: build
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          # fetch the whole repot for `git describe` to
+          # work and get the nightly verion
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+
+      - name: Make Release
+        env:
+          GOBIN: /usr/local/bin
+        run: |
+          make release
+
+      - name: Publish nightly release to aptly
+        if: github.ref == 'refs/heads/master'
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh nightly
+
+      - name: Publish release to aptly
+        if: startsWith(github.event.ref, 'refs/tags')
+        env:
+          APTLY_USER: ${{ secrets.APTLY_USER }}
+          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
+        run: |
+          ./upload-artifacts.sh release
+
+      - name: Upload artifacts to GitHub Release
+        if: startsWith(github.event.ref, 'refs/tags')
+        uses: softprops/action-gh-release@v1
+        with:
+          body: Release ${{ github.ref }} generated by the CI.
+          files: build/*

.gitignore

@@ -41,3 +41,4 @@ build/
 
 pgp/keyrings/aptly2*.gpg
 pgp/keyrings/aptly2*.gpg~
+pgp/keyrings/.#*

.golangci.yml

@@ -3,20 +3,17 @@ run:
 
 
 linters:
-  enable-all: false
   disable-all: true
   enable:
-   - govet
-   - golint
-   - gofmt
    - deadcode
-   - goimports
-   - misspell
-   - ineffassign
-   - staticcheck
-   - varcheck
-   - structcheck
-   - maligned
-   - vetshadow
    - goconst
-   - interfacer
+   - gofmt
+   - goimports
+   - govet
+   - ineffassign
+   - misspell
+   - revive
+   - staticcheck
+   - structcheck
+   - varcheck
+   - vetshadow

.goxc.json

@@ -34,8 +34,7 @@
         }
     },
 	"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
-	"Arch": "386 amd64",
-	"Os": "linux darwin freebsd",
+	"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
 	"MainDirsExclude": "_man,vendor",
 	"BuildSettings": {
 		"LdFlagsXVars": {

.travis.yml

@@ -1,4 +1,4 @@
-dist: trusty
+dist: xenial
 sudo: required
 
 language: go
@@ -19,6 +19,8 @@ env:
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
     - NO_FTP_ACCESS: "yes"
     - BOTO_CONFIG: /dev/null
+    - GO111MODULE: "on"
+    - GOPROXY: https://proxy.golang.org
 
 matrix:
   allow_failures:
@@ -26,11 +28,15 @@ matrix:
       env: RUN_LONG_TESTS=no
   fast_finish: true
   include:
-    - go: 1.10.x
-      env: RUN_LONG_TESTS=no
     - go: 1.11.x
-      env: RUN_LONG_TESTS=yes
+      env: RUN_LONG_TESTS=no
     - go: 1.12.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.13.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.14.x
+      env: RUN_LONG_TESTS=yes
+    - go: 1.15.x
       env:
         - RUN_LONG_TESTS=yes
         - DEPLOY_BINARIES=yes

AUTHORS

@@ -35,3 +35,17 @@ List of contributors, in chronological order:
 * Strajan Sebastian (https://github.com/strajansebastian)
 * Artem Smirnov (https://github.com/urpylka)
 * William Manley (https://github.com/wmanley)
+* Shengjing Zhu (https://github.com/zhsj)
+* Nabil Bendafi (https://github.com/nabilbendafi)
+* Raphael Medaer (https://github.com/rmedaer)
+* Raul Benencia (https://github.com/rul)
+* Don Kuntz (https://github.com/dkuntz2)
+* Joshua Colson (https://github.com/freakinhippie)
+* Andre Roth (https://github.com/neolynx)
+* Lorenzo Bolla (https://github.com/lbolla)
+* Benj Fassbind (https://github.com/randombenj)
+* Markus Muellner (https://github.com/mmianl)
+* Chuan Liu (https://github.com/chuan)
+* Samuel Mutel (https://github.com/smutel)
+* Russell Greene (https://github.com/russelltg)
+* Wade Simmons (https://github.com/wadey)

CONTRIBUTING.md

@@ -40,7 +40,7 @@ Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info)
 
 There are two kinds of documentation:
 
-* [aptly website](https://www.aptly/info)
+* [aptly website](https://www.aptly.info)
 * aptly `man` page
 
 Core content is mostly the same, but website contains more information, tutorials, examples.
@@ -84,10 +84,8 @@ to prepend it or to skip this test if you're security conscious.
 
 ### Forking and Cloning
 
-As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
+As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
 
-    mkdir -p ~/go/src/github.com/aptly-dev
-    cd ~/go/src/github.com/aptly-dev
     git clone git@github.com:aptly-dev/aptly.git
     cd aptly
 
@@ -108,6 +106,10 @@ You would need some additional tools and Python virtual environment to run tests
 
 This is usually one-time action.
 
+Aptly is using Go modules to manage dependencies, download modules using:
+
+    make modules
+
 ### Building
 
 If you want to build aptly binary from your current source tree, run:

Gopkg.lock

@@ -1,404 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:e8777c437e157121465a08c05304d5847ae70b5683ef4afeb723b3c9e5e3bc67"
-  name = "github.com/AlekSi/pointer"
-  packages = ["."]
-  pruneopts = ""
-  revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:218a66570875d243938fff08f17ec03e79b18c138d0845b2428bcb4929ffa4fe"
-  name = "github.com/DisposaBoy/JsonConfigReader"
-  packages = ["."]
-  pruneopts = ""
-  revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
-
-[[projects]]
-  digest = "1:658ab137074ef3d1216e01cbe166eb171a1bd77daeae0b686ab1ae184a9a5ec9"
-  name = "github.com/awalterschulze/gographviz"
-  packages = [
-    ".",
-    "ast",
-    "parser",
-    "scanner",
-    "token"
-  ]
-  pruneopts = ""
-  revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
-  version = "v1.0"
-
-[[projects]]
-  digest = "1:a72e35a51b628d148241720897de644fd6d3cea45d8489dae9d373ced5dfc302"
-  name = "github.com/aws/aws-sdk-go"
-  packages = [
-    "aws",
-    "aws/awserr",
-    "aws/awsutil",
-    "aws/client",
-    "aws/client/metadata",
-    "aws/corehandlers",
-    "aws/credentials",
-    "aws/credentials/ec2rolecreds",
-    "aws/credentials/endpointcreds",
-    "aws/credentials/stscreds",
-    "aws/defaults",
-    "aws/ec2metadata",
-    "aws/endpoints",
-    "aws/request",
-    "aws/session",
-    "aws/signer/v4",
-    "internal/sdkio",
-    "internal/sdkrand",
-    "internal/shareddefaults",
-    "private/protocol",
-    "private/protocol/query",
-    "private/protocol/query/queryutil",
-    "private/protocol/rest",
-    "private/protocol/restxml",
-    "private/protocol/xml/xmlutil",
-    "service/s3",
-    "service/sts"
-  ]
-  pruneopts = ""
-  revision = "a72204b9bf8d48230ee0fe8995613b394c66f2da"
-  version = "v1.13.31"
-
-[[projects]]
-  digest = "1:b9922c7da8a89c973758c03bde497017567f785b49bd253402a3587abec1e53d"
-  name = "github.com/cheggaaa/pb"
-  packages = ["."]
-  pruneopts = ""
-  revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
-  version = "v1.0.10"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1120f960f5c334f0f94bad29eefaf73d52d226893369693686148f66c1993f15"
-  name = "github.com/gin-contrib/sse"
-  packages = ["."]
-  pruneopts = ""
-  revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
-
-[[projects]]
-  digest = "1:348ceb76f2ac958e541e4ba3190484b68df28c38ac9720ed4ef8d36af69ce52e"
-  name = "github.com/gin-gonic/gin"
-  packages = [
-    ".",
-    "binding",
-    "render"
-  ]
-  pruneopts = ""
-  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
-  version = "v1.2"
-
-[[projects]]
-  digest = "1:fbed9ba4076145ae05ef7deeb14d49565e112d6c283a8c4304fa3b7be785fa5e"
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  pruneopts = ""
-  revision = "1730955e3146956d6a087861380f9b4667ed5071"
-  version = "v1.26.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:27854310d59099f8dcc61dd8af4a69f0a3597f001154b2fb4d1c41baf2e31ec1"
-  name = "github.com/golang/protobuf"
-  packages = ["proto"]
-  pruneopts = ""
-  revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
-
-[[projects]]
-  branch = "master"
-  digest = "1:09307dfb1aa3f49a2bf869dcfa4c6c06ecd3c207221bd1c1a1141f0e51f209eb"
-  name = "github.com/golang/snappy"
-  packages = ["."]
-  pruneopts = ""
-  revision = "553a641470496b2327abcac10b36396bd98e45c9"
-
-[[projects]]
-  digest = "1:6ee50e0ace655f26787c120dbce51b3185c470f475c9e57d0dcada8ebb115004"
-  name = "github.com/h2non/filetype"
-  packages = ["matchers"]
-  pruneopts = ""
-  revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb"
-  version = "v1.0.5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:64b78d98b8956492576911baf6a1e3499816d4575e485d12792e4abe7d8b6c46"
-  name = "github.com/jlaffaye/ftp"
-  packages = ["."]
-  pruneopts = ""
-  revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
-
-[[projects]]
-  digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e"
-  name = "github.com/jmespath/go-jmespath"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0b12d6b5"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c728183dd470c8bb2d1775edc641a81ec25f5e3804b8b0536e99f71df17a13a6"
-  name = "github.com/kjk/lzma"
-  packages = ["."]
-  pruneopts = ""
-  revision = "3fd93898850d5252457e48c1b3d5e1510597280b"
-
-[[projects]]
-  digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
-  name = "github.com/mattn/go-runewidth"
-  packages = ["."]
-  pruneopts = ""
-  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
-  version = "v0.0.2"
-
-[[projects]]
-  digest = "1:fb8502ed69803c4cd97b1def119db73869ec2f7e2b408eb9c1b8276df6f05b3e"
-  name = "github.com/mattn/go-shellwords"
-  packages = ["."]
-  pruneopts = ""
-  revision = "005a0944d84452842197c2108bd9168ced206f78"
-  version = "v1.0.2"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3ba28ef4fbbf8d099c6227b698a6ceae28f8a071e12f06839c12c4c0003b7554"
-  name = "github.com/mkrautz/goar"
-  packages = ["."]
-  pruneopts = ""
-  revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
-
-[[projects]]
-  branch = "master"
-  digest = "1:d33ce379780d7c43405b9251289493cabada82f6bf9ab35eea6915d04f6ac8e0"
-  name = "github.com/mxk/go-flowrate"
-  packages = ["flowrate"]
-  pruneopts = ""
-  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
-
-[[projects]]
-  branch = "master"
-  digest = "1:428d8af27f534ed06d783b03d477124796de06aa86402777cd2b494c64278da5"
-  name = "github.com/ncw/swift"
-  packages = [
-    ".",
-    "swifttest"
-  ]
-  pruneopts = ""
-  revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:beeb9206cc21cfeb113066c3dcf4bbb0ba304d73dd441f3244721566f51f44e6"
-  name = "github.com/pborman/uuid"
-  packages = ["."]
-  pruneopts = ""
-  revision = "c65b2f87fee37d1c7854c9164a450713c28d50cd"
-
-[[projects]]
-  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = ""
-  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
-  version = "v0.8.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c6e71685a14409c3a1fdbecef044ee6447c3887de7c2bb025ee12f9a8a368960"
-  name = "github.com/smira/commander"
-  packages = ["."]
-  pruneopts = ""
-  revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a75eb2e6e718988505f704d651eeb4734f0f0b580f761aa58b4a5b0afe585091"
-  name = "github.com/smira/flag"
-  packages = ["."]
-  pruneopts = ""
-  revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cd3b8057abbb453404cbef569951a7359986eedfc0a887f9bc5efff8d02e1760"
-  name = "github.com/smira/go-aws-auth"
-  packages = ["."]
-  pruneopts = ""
-  revision = "8b73995fd8d1d82519c7037ee243a497552cd54d"
-
-[[projects]]
-  branch = "master"
-  digest = "1:7dd59b3536a07651baf394466da3b8a20bd3ed9ce1984c7583fdf2412fdcfea9"
-  name = "github.com/smira/go-ftp-protocol"
-  packages = ["protocol"]
-  pruneopts = ""
-  revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
-
-[[projects]]
-  branch = "master"
-  digest = "1:55fe1086100b305a5562e9f1e6825ef97c49e82331cb0f055b2069f7c7ebd469"
-  name = "github.com/smira/go-xz"
-  packages = ["."]
-  pruneopts = ""
-  revision = "0c531f070014e218b21f3cfca801cc992d52726d"
-
-[[projects]]
-  branch = "master"
-  digest = "1:175bc8d87d54849b9b9fc6cd473c5830e090fbf3c2f0ca71c7642a697e5d9237"
-  name = "github.com/syndtr/goleveldb"
-  packages = [
-    "leveldb",
-    "leveldb/cache",
-    "leveldb/comparer",
-    "leveldb/errors",
-    "leveldb/filter",
-    "leveldb/iterator",
-    "leveldb/journal",
-    "leveldb/memdb",
-    "leveldb/opt",
-    "leveldb/storage",
-    "leveldb/table",
-    "leveldb/util"
-  ]
-  pruneopts = ""
-  revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
-
-[[projects]]
-  digest = "1:b9bca27d5fbe4ad1a1802706629580ce923bb14255c51375480dff1bebcbb8b2"
-  name = "github.com/ugorji/go"
-  packages = ["codec"]
-  pruneopts = ""
-  revision = "71c2886f5a673a35f909803f38ece5810165097b"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2d8078b329a80bf4b71537dcd30853ff89fcafb611c94b2dfdb3d43ef1f23a1a"
-  name = "github.com/wsxiaoys/terminal"
-  packages = ["color"]
-  pruneopts = ""
-  revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
-
-[[projects]]
-  branch = "master"
-  digest = "1:51aff2b69272cb95713b74675c0817e0a0caadc20337a8d9115bc68bf1105280"
-  name = "golang.org/x/crypto"
-  packages = [
-    "cast5",
-    "openpgp",
-    "openpgp/armor",
-    "openpgp/clearsign",
-    "openpgp/elgamal",
-    "openpgp/errors",
-    "openpgp/packet",
-    "openpgp/s2k",
-    "ssh/terminal"
-  ]
-  pruneopts = ""
-  revision = "b2aa35443fbc700ab74c586ae79b81c171851023"
-
-[[projects]]
-  branch = "master"
-  digest = "1:267aa6124260d0e48d1ebf8bb84dce97caa133406b3caabc076cfc7a9e85ffe4"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows"
-  ]
-  pruneopts = ""
-  revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e"
-
-[[projects]]
-  branch = "v1"
-  digest = "1:e75566abfb876e81f00290ec153ff994c33bf8886134c1a38a9a9df5c15a2045"
-  name = "gopkg.in/check.v1"
-  packages = ["."]
-  pruneopts = ""
-  revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
-
-[[projects]]
-  digest = "1:dd549e360e5a8f982a28c2bcbe667307ceffe538ed9afc7c965524f1ac285b3f"
-  name = "gopkg.in/go-playground/validator.v8"
-  packages = ["."]
-  pruneopts = ""
-  revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
-  version = "v8.18.2"
-
-[[projects]]
-  digest = "1:0e886d5a5845f58255fb57bcb4a6aae7e1f9dd7c53defe7b87d57dd4cd127545"
-  name = "gopkg.in/h2non/filetype.v1"
-  packages = ["types"]
-  pruneopts = ""
-  revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = ""
-  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/AlekSi/pointer",
-    "github.com/DisposaBoy/JsonConfigReader",
-    "github.com/awalterschulze/gographviz",
-    "github.com/aws/aws-sdk-go/aws",
-    "github.com/aws/aws-sdk-go/aws/awserr",
-    "github.com/aws/aws-sdk-go/aws/corehandlers",
-    "github.com/aws/aws-sdk-go/aws/credentials",
-    "github.com/aws/aws-sdk-go/aws/request",
-    "github.com/aws/aws-sdk-go/aws/session",
-    "github.com/aws/aws-sdk-go/service/s3",
-    "github.com/cheggaaa/pb",
-    "github.com/gin-gonic/gin",
-    "github.com/h2non/filetype/matchers",
-    "github.com/kjk/lzma",
-    "github.com/mattn/go-shellwords",
-    "github.com/mkrautz/goar",
-    "github.com/mxk/go-flowrate/flowrate",
-    "github.com/ncw/swift",
-    "github.com/ncw/swift/swifttest",
-    "github.com/pborman/uuid",
-    "github.com/pkg/errors",
-    "github.com/smira/commander",
-    "github.com/smira/flag",
-    "github.com/smira/go-aws-auth",
-    "github.com/smira/go-ftp-protocol/protocol",
-    "github.com/smira/go-xz",
-    "github.com/syndtr/goleveldb/leveldb",
-    "github.com/syndtr/goleveldb/leveldb/filter",
-    "github.com/syndtr/goleveldb/leveldb/opt",
-    "github.com/syndtr/goleveldb/leveldb/storage",
-    "github.com/syndtr/goleveldb/leveldb/util",
-    "github.com/ugorji/go/codec",
-    "github.com/wsxiaoys/terminal/color",
-    "golang.org/x/crypto/openpgp",
-    "golang.org/x/crypto/openpgp/armor",
-    "golang.org/x/crypto/openpgp/clearsign",
-    "golang.org/x/crypto/openpgp/errors",
-    "golang.org/x/crypto/openpgp/packet",
-    "golang.org/x/crypto/ssh/terminal",
-    "golang.org/x/sys/unix",
-    "gopkg.in/check.v1"
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,28 +0,0 @@
-
-[[override]]
-  branch = "master"
-  name = "github.com/mkrautz/goar"
-
-[[override]]
-  branch = "master"
-  name = "github.com/smira/go-uuid"
-
-[[override]]
-  branch = "master"
-  name = "github.com/smira/go-xz"
-
-[[override]]
-  name = "github.com/ugorji/go"
-  revision = "71c2886f5a673a35f909803f38ece5810165097b"
-
-[[override]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[[override]]
-  branch = "master"
-  name = "golang.org/x/sys"
-
-[[override]]
-  branch = "v1"
-  name = "gopkg.in/check.v1"

Makefile

@@ -2,28 +2,32 @@ GOVERSION=$(shell go version | awk '{print $$3;}')
 ifdef TRAVIS_TAG
 	TAG=$(TRAVIS_TAG)
 else
-	TAG="$(shell git describe --tags)"
+	TAG="$(shell git describe --tags --always)"
 endif
 VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
 PACKAGES=context database deb files gpg http query swift s3 utils
-PYTHON?=python
+PYTHON?=python3
 TESTS?=
 BINPATH?=$(GOPATH)/bin
 RUN_LONG_TESTS?=yes
 
-all: test bench check system-test
+all: modules test bench check system-test
 
 prepare:
-	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.17.1
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+
+modules:
+	go mod download
+	go mod verify
+	go mod tidy -v
 
 dev:
-	go get -u github.com/golang/dep/...
 	go get -u github.com/laher/goxc
 
 check: system/env
 ifeq ($(RUN_LONG_TESTS), yes)
 	golangci-lint run
-	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
+	system/env/bin/flake8
 endif
 
 install:
@@ -32,7 +36,7 @@ install:
 system/env: system/requirements.txt
 ifeq ($(RUN_LONG_TESTS), yes)
 	rm -rf system/env
-	virtualenv system/env
+	$(PYTHON) -m venv system/env
 	system/env/bin/pip install -r system/requirements.txt
 endif
 
@@ -74,4 +78,4 @@ man:
 version:
 	@echo $(VERSION)
 
-.PHONY: man version release goxc
+.PHONY: man modules version release goxc

README.rst

@@ -2,17 +2,17 @@
 aptly
 =====
 
-.. image:: https://api.travis-ci.org/aptly-dev/aptly.svg?branch=master
-    :target: https://travis-ci.org/aptly-dev/aptly
+.. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/aptly-dev/aptly/actions
 
 .. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
-  :target: https://codecov.io/gh/aptly-dev/aptly
+    :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
     :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: http://goreportcard.com/badge/aptly-dev/aptly
-    :target: http://goreportcard.com/report/aptly-dev/aptly
+.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
+    :target: https://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -48,7 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list
 
 And import key that is used to sign the release::
 
-    $ apt-key adv --keyserver pool.sks-keyservers.net --recv-keys ED75B5A4483DA07C
+    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
 
 After that you can install aptly as any other software package::
 
@@ -62,14 +62,13 @@ If you would like to use nightly builds (unstable), please use following reposit
 
     deb http://repo.aptly.info/ nightly main
 
-Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
+Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.10+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.14+ required)::
 
-    mkdir -p $GOPATH/src/github.com/aptly-dev/aptly
-    git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly
-    cd $GOPATH/src/github.com/aptly-dev/aptly
-    make install
+    git clone https://github.com/aptly-dev/aptly
+    cd aptly
+    make modules install
 
 Binary would be installed to ``$GOPATH/bin/aptly``.
 

api/api.go

@@ -3,12 +3,16 @@ package api
 
 import (
 	"fmt"
+	"log"
+	"net/http"
 	"sort"
-	"time"
+	"strconv"
+	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
@@ -35,49 +39,14 @@ type dbRequest struct {
 	err  chan<- error
 }
 
-// Flushes all collections which cache in-memory objects
-func flushColections() {
-	// lock everything to eliminate in-progress calls
-	r := context.CollectionFactory().RemoteRepoCollection()
-	r.Lock()
-	defer r.Unlock()
-
-	l := context.CollectionFactory().LocalRepoCollection()
-	l.Lock()
-	defer l.Unlock()
-
-	s := context.CollectionFactory().SnapshotCollection()
-	s.Lock()
-	defer s.Unlock()
-
-	p := context.CollectionFactory().PublishedRepoCollection()
-	p.Lock()
-	defer p.Unlock()
-
-	// all collections locked, flush them
-	context.CollectionFactory().Flush()
-}
-
-// Periodically flushes CollectionFactory to free up memory used by
-// collections, flushing caches.
-//
-// Should be run in goroutine!
-func cacheFlusher() {
-	ticker := time.Tick(15 * time.Minute)
-
-	for {
-		<-ticker
-
-		flushColections()
-	}
-}
+var dbRequests chan dbRequest
 
 // Acquire database lock and release it when not needed anymore.
 //
 // Should be run in a goroutine!
-func acquireDatabase(requests <-chan dbRequest) {
+func acquireDatabase() {
 	clients := 0
-	for request := range requests {
+	for request := range dbRequests {
 		var err error
 
 		switch request.kind {
@@ -94,7 +63,6 @@ func acquireDatabase(requests <-chan dbRequest) {
 		case releasedb:
 			clients--
 			if clients == 0 {
-				flushColections()
 				err = context.CloseDatabase()
 			} else {
 				err = nil
@@ -105,12 +73,100 @@ func acquireDatabase(requests <-chan dbRequest) {
 	}
 }
 
+// Should be called before database access is needed in any api call.
+// Happens per default for each api call. It is important that you run
+// runTaskInBackground to run a task which accquire database.
+// Important do not forget to defer to releaseDatabaseConnection
+func acquireDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{acquiredb, errCh}
+
+	return <-errCh
+}
+
+// Release database connection when not needed anymore
+func releaseDatabaseConnection() error {
+	if dbRequests == nil {
+		return nil
+	}
+
+	errCh := make(chan error)
+	dbRequests <- dbRequest{releasedb, errCh}
+	return <-errCh
+}
+
+// runs tasks in background. Acquires database connection first.
+func runTaskInBackground(name string, resources []string, proc task.Process) (task.Task, *task.ResourceConflictError) {
+	return context.TaskList().RunTaskInBackground(name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := acquireDatabaseConnection()
+
+		if err != nil {
+			return nil, err
+		}
+
+		defer releaseDatabaseConnection()
+		return proc(out, detail)
+	})
+}
+
+func truthy(value interface{}) bool {
+	if value == nil {
+		return false
+	}
+	switch value.(type) {
+	case string:
+		switch strings.ToLower(value.(string)) {
+		case "n", "no", "f", "false", "0", "off":
+			return false
+		default:
+			return true
+		}
+	case int:
+		return !(value.(int) == 0)
+	case bool:
+		return value.(bool)
+	}
+	return true
+}
+
+func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, proc task.Process) {
+	// Run this task in background if configured globally or per-request
+	background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
+	if background {
+		log.Println("Executing task asynchronously")
+		task, conflictErr := runTaskInBackground(name, resources, proc)
+		if conflictErr != nil {
+			c.AbortWithError(409, conflictErr)
+			return
+		}
+		c.JSON(202, task)
+	} else {
+		log.Println("Executing task synchronously")
+		out := context.Progress()
+		detail := task.Detail{}
+		retValue, err := proc(out, &detail)
+		if err != nil {
+			c.AbortWithError(retValue.Code, err)
+			return
+		}
+		if retValue != nil {
+			c.JSON(retValue.Code, retValue.Value)
+		} else {
+			c.JSON(http.StatusOK, nil)
+		}
+	}
+}
+
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return

api/api_test.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"encoding/json"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	TestingT(t)
+}
+
+type ApiSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+}
+
+var _ = Suite(&ApiSuite{})
+
+func createTestConfig() *os.File {
+	file, err := ioutil.TempFile("", "aptly")
+	if err != nil {
+		return nil
+	}
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+	})
+	if err != nil {
+		return nil
+	}
+	file.Write(jsonString)
+	return file
+}
+
+func (s *ApiSuite) SetUpSuite(c *C) {
+	file := createTestConfig()
+	c.Assert(file, NotNil)
+	s.configFile = file
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *ApiSuite) TearDownSuite(c *C) {
+	os.Remove(s.configFile.Name())
+	s.context.Shutdown()
+}
+
+func (s *ApiSuite) SetUpTest(c *C) {
+}
+
+func (s *ApiSuite) TearDownTest(c *C) {
+}
+
+func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+	w := httptest.NewRecorder()
+	req, err := http.NewRequest(method, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w, nil
+}
+
+func (s *ApiSuite) TestGetVersion(c *C) {
+	response, err := s.HTTPRequest("GET", "/api/version", nil)
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, ".*Version.*")
+}
+
+func (s *ApiSuite) TestTruthy(c *C) {
+	c.Check(truthy("no"), Equals, false)
+	c.Check(truthy("n"), Equals, false)
+	c.Check(truthy("off"), Equals, false)
+	c.Check(truthy("false"), Equals, false)
+	c.Check(truthy("0"), Equals, false)
+	c.Check(truthy(false), Equals, false)
+	c.Check(truthy(0), Equals, false)
+
+	c.Check(truthy("y"), Equals, true)
+	c.Check(truthy("yes"), Equals, true)
+	c.Check(truthy("t"), Equals, true)
+	c.Check(truthy("true"), Equals, true)
+	c.Check(truthy("1"), Equals, true)
+	c.Check(truthy(true), Equals, true)
+	c.Check(truthy(1), Equals, true)
+
+	c.Check(truthy(nil), Equals, false)
+
+	c.Check(truthy("foobar"), Equals, true)
+	c.Check(truthy(-1), Equals, true)
+	c.Check(truthy(gin.H{}), Equals, true)
+}

api/db.go

@@ -0,0 +1,180 @@
+package api
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/db/cleanup
+func apiDbCleanup(c *gin.Context) {
+
+	resources := []string{string(task.AllResourcesKey)}
+	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var err error
+
+		collectionFactory := context.NewCollectionFactory()
+
+		// collect information about referenced packages...
+		existingPackageRefs := deb.NewPackageRefList()
+
+		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
+		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+			if e != nil {
+				return e
+			}
+
+			if repo.RefList() != nil {
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+			if e != nil {
+				return e
+			}
+
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+			if published.SourceKind != deb.SourceLocalRepo {
+				return nil
+			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
+
+			for _, component := range published.Components() {
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// ... and compare it to the list of all packages
+		out.Printf("Loading list of all packages...")
+		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
+
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
+
+		// delete packages that are no longer referenced
+		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
+
+		// database can't err as collection factory already constructed
+		db, _ := context.Database()
+
+		if toDelete.Len() > 0 {
+			batch := db.CreateBatch()
+			toDelete.ForEach(func(ref []byte) error {
+				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				return nil
+			})
+
+			err = batch.Write()
+			if err != nil {
+				return nil, fmt.Errorf("unable to write to DB: %s", err)
+			}
+		}
+
+		// now, build a list of files that should be present in Repository (package pool)
+		out.Printf("Building list of files referenced by packages...")
+		referencedFiles := make([]string, 0, existingPackageRefs.Len())
+
+		err = existingPackageRefs.ForEach(func(key []byte) error {
+			pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
+			if err2 != nil {
+				tail := ""
+				return fmt.Errorf("unable to load package %s: %s%s", string(key), err2, tail)
+			}
+			paths, err2 := pkg.FilepathList(context.PackagePool())
+			if err2 != nil {
+				return err2
+			}
+			referencedFiles = append(referencedFiles, paths...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		sort.Strings(referencedFiles)
+
+		// build a list of files in the package pool
+		out.Printf("Building list of files in package pool...")
+		existingFiles, err := context.PackagePool().FilepathList(out)
+		if err != nil {
+			return nil, fmt.Errorf("unable to collect file paths: %s", err)
+		}
+
+		// find files which are in the pool but not referenced by packages
+		filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+		// delete files that are no longer referenced
+		out.Printf("Deleting unreferenced files (%d)...", len(filesToDelete))
+
+		countFilesToDelete := len(filesToDelete)
+		taskDetail := struct {
+			TotalNumberOfPackagesToDelete     int
+			RemainingNumberOfPackagesToDelete int
+		}{
+			countFilesToDelete, countFilesToDelete,
+		}
+		detail.Store(taskDetail)
+
+		if countFilesToDelete > 0 {
+			var size, totalSize int64
+			for _, file := range filesToDelete {
+				size, err = context.PackagePool().Remove(file)
+				if err != nil {
+					return nil, err
+				}
+
+				taskDetail.RemainingNumberOfPackagesToDelete--
+				detail.Store(taskDetail)
+				totalSize += size
+			}
+
+			out.Printf("Disk space freed: %s...", utils.HumanBytes(totalSize))
+		}
+
+		out.Printf("Compacting database...")
+		return nil, db.CompactDB()
+	})
+}

api/gpg.go

@@ -0,0 +1,84 @@
+package api
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/gin-gonic/gin"
+)
+
+// POST /api/gpg
+func apiGPGAddKey(c *gin.Context) {
+	var b struct {
+		Keyserver   string
+		GpgKeyID    string
+		GpgKeyArmor string
+		Keyring     string
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	var err error
+	args := []string{"--no-default-keyring"}
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+	args = append(args, "--keyring", keyring)
+	if len(b.Keyserver) > 0 {
+		args = append(args, "--keyserver", b.Keyserver)
+	}
+	if len(b.GpgKeyArmor) > 0 {
+		var tempdir string
+		tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+		defer os.RemoveAll(tempdir)
+
+		keypath := filepath.Join(tempdir, "key")
+		keyfile, e := os.Create(keypath)
+		if e != nil {
+			c.AbortWithError(400, e)
+			return
+		}
+		if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
+			c.AbortWithError(400, e)
+		}
+		args = append(args, "--import", keypath)
+
+	}
+	if len(b.GpgKeyID) > 0 {
+		keys := strings.Fields(b.GpgKeyID)
+		args = append(args, "--recv-keys")
+		args = append(args, keys...)
+	}
+
+	finder := pgp.GPG1Finder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	// it might happened that we have a situation with an erroneous
+	// gpg command (e.g. when GpgKeyID and GpgKeyArmor is set).
+	// there is no error handling for such as gpg will do this for us
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	cmd.Stdout = os.Stdout
+	if err = cmd.Run(); err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, gin.H{})
+}

api/graph.go

@@ -21,17 +21,7 @@ func apiGraph(c *gin.Context) {
 
 	ext := c.Params.ByName("ext")
 	layout := c.Request.URL.Query().Get("layout")
-
-	factory := context.CollectionFactory()
-
-	factory.RemoteRepoCollection().Lock()
-	defer factory.RemoteRepoCollection().Unlock()
-	factory.LocalRepoCollection().Lock()
-	defer factory.LocalRepoCollection().Unlock()
-	factory.SnapshotCollection().Lock()
-	defer factory.SnapshotCollection().Unlock()
-	factory.PublishedRepoCollection().Lock()
-	defer factory.PublishedRepoCollection().Unlock()
+	factory := context.NewCollectionFactory()
 
 	graph, err := deb.BuildGraph(factory, layout)
 	if err != nil {

api/middleware.go

@@ -0,0 +1,103 @@
+package api
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+var (
+	apiRequestsInFlightGauge = promauto.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "aptly_api_http_requests_in_flight",
+			Help: "Number of concurrent HTTP api requests currently handled.",
+		},
+		[]string{"method", "path"},
+	)
+	apiRequestsTotalCounter = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "aptly_api_http_requests_total",
+			Help: "Total number of api requests.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_size_bytes",
+			Help: "Api HTTP request size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiResponseSizeSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_response_size_bytes",
+			Help: "Api HTTP response size in bytes.",
+		},
+		[]string{"code", "method", "path"},
+	)
+	apiRequestsDurationSummary = promauto.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: "aptly_api_http_request_duration_seconds",
+			Help: "Duration of api requests in seconds.",
+		},
+		[]string{"code", "method", "path"},
+	)
+)
+
+// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
+// See https://prometheus.io/docs/practices/naming/#labels
+func getBasePath(c *gin.Context) string {
+	return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
+}
+
+func getURLSegment(url string, idx int) string {
+	var urlSegments = strings.Split(url, "/")
+
+	// Remove segment at index 0 because it's an empty string
+	var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
+	return fmt.Sprintf("/%s", segmentAtIndex)
+}
+
+func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		g.WithLabelValues(c.Request.Method, pathFunc(c)).Inc()
+		defer g.WithLabelValues(c.Request.Method, pathFunc(c)).Dec()
+		c.Next()
+	}
+}
+
+func instrumentHandlerCounter(counter *prometheus.CounterVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		counter.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Inc()
+	}
+}
+
+func instrumentHandlerRequestSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(float64(c.Request.ContentLength))
+	}
+}
+
+func instrumentHandlerResponseSize(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		c.Next()
+		var responseSize = math.Max(float64(c.Writer.Size()), 0)
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(responseSize)
+	}
+}
+
+func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
+	return func(c *gin.Context) {
+		now := time.Now()
+		c.Next()
+		obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
+	}
+}

api/mirror.go

@@ -0,0 +1,556 @@
+package api
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
+	if ignoreSignatures {
+		return nil, nil
+	}
+
+	verifier := context.GetVerifier()
+	for _, keyRing := range keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring()
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+// GET /api/mirrors
+func apiMirrorsList(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	result := []*deb.RemoteRepo{}
+	collection.ForEach(func(repo *deb.RemoteRepo) error {
+		result = append(result, repo)
+		return nil
+	})
+
+	c.JSON(200, result)
+}
+
+// POST /api/mirrors
+func apiMirrorsCreate(c *gin.Context) {
+	var err error
+	var b struct {
+		Name               string `binding:"required"`
+		ArchiveURL         string `binding:"required"`
+		Distribution       string
+		Filter             string
+		Components         []string
+		Architectures      []string
+		Keyrings           []string
+		DownloadSources    bool
+		DownloadUdebs      bool
+		DownloadInstaller  bool
+		FilterWithDeps     bool
+		SkipComponentCheck bool
+		IgnoreSignatures   bool
+	}
+
+	b.DownloadSources = context.Config().DownloadSourcePackages
+	b.IgnoreSignatures = context.Config().GpgDisableVerify
+	b.Architectures = context.ArchitecturesList()
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	if strings.HasPrefix(b.ArchiveURL, "ppa:") {
+		b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+	}
+
+	if b.Filter != "" {
+		_, err = query.Parse(b.Filter)
+		if err != nil {
+			c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+			return
+		}
+	}
+
+	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
+		return
+	}
+
+	repo.Filter = b.Filter
+	repo.FilterWithDeps = b.FilterWithDeps
+	repo.SkipComponentCheck = b.SkipComponentCheck
+	repo.DownloadSources = b.DownloadSources
+	repo.DownloadUdebs = b.DownloadUdebs
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	downloader := context.NewDownloader(nil)
+	err = repo.Fetch(downloader, verifier)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
+		return
+	}
+
+	err = collection.Add(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
+		return
+	}
+
+	c.JSON(201, repo)
+}
+
+// DELETE /api/mirrors/:name
+func apiMirrorsDrop(c *gin.Context) {
+	name := c.Params.ByName("name")
+	force := c.Request.URL.Query().Get("force") == "1"
+
+	collectionFactory := context.NewCollectionFactory()
+	mirrorCollection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+
+	repo, err := mirrorCollection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
+		return
+	}
+
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		if !force {
+			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
+			}
+		}
+
+		err = mirrorCollection.Drop(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}
+
+// GET /api/mirrors/:name
+func apiMirrorsShow(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	c.JSON(200, repo)
+}
+
+// GET /api/mirrors/:name/packages
+func apiMirrorsPackages(c *gin.Context) {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
+	if err != nil {
+		c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
+		return
+	}
+
+	err = collection.LoadComplete(repo)
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
+	}
+
+	if repo.LastDownloadDate.IsZero() {
+		c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
+		return
+	}
+
+	reflist := repo.RefList()
+	result := []*deb.Package{}
+
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	queryS := c.Request.URL.Query().Get("q")
+	if queryS != "" {
+		q, err := query.Parse(c.Request.URL.Query().Get("q"))
+		if err != nil {
+			c.AbortWithError(400, err)
+			return
+		}
+
+		withDeps := c.Request.URL.Query().Get("withDeps") == "1"
+		architecturesList := []string{}
+
+		if withDeps {
+			if len(context.ArchitecturesList()) > 0 {
+				architecturesList = context.ArchitecturesList()
+			} else {
+				architecturesList = list.Architectures(false)
+			}
+
+			sort.Strings(architecturesList)
+
+			if len(architecturesList) == 0 {
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				return
+			}
+		}
+
+		list.PrepareIndex()
+
+		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
+			nil, context.DependencyOptions(), architecturesList)
+		if err != nil {
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
+		}
+	}
+
+	if c.Request.URL.Query().Get("format") == "details" {
+		list.ForEach(func(p *deb.Package) error {
+			result = append(result, p)
+			return nil
+		})
+
+		c.JSON(200, result)
+	} else {
+		c.JSON(200, list.Strings())
+	}
+}
+
+// PUT /api/mirrors/:name
+func apiMirrorsUpdate(c *gin.Context) {
+	var (
+		err    error
+		remote *deb.RemoteRepo
+	)
+
+	var b struct {
+		Name                 string
+		ArchiveURL           string
+		Filter               string
+		Architectures        []string
+		Components           []string
+		Keyrings             []string
+		FilterWithDeps       bool
+		DownloadSources      bool
+		DownloadUdebs        bool
+		SkipComponentCheck   bool
+		IgnoreChecksums      bool
+		IgnoreSignatures     bool
+		ForceUpdate          bool
+		SkipExistingPackages bool
+	}
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	remote, err = collection.ByName(c.Params.ByName("name"))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	b.Name = remote.Name
+	b.DownloadUdebs = remote.DownloadUdebs
+	b.DownloadSources = remote.DownloadSources
+	b.SkipComponentCheck = remote.SkipComponentCheck
+	b.FilterWithDeps = remote.FilterWithDeps
+	b.Filter = remote.Filter
+	b.Architectures = remote.Architectures
+	b.Components = remote.Components
+
+	log.Printf("%s: Starting mirror update\n", b.Name)
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	if b.Name != remote.Name {
+		_, err = collection.ByName(b.Name)
+		if err == nil {
+			c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
+			return
+		}
+	}
+
+	if b.DownloadUdebs != remote.DownloadUdebs {
+		if remote.IsFlat() && b.DownloadUdebs {
+			c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
+			return
+		}
+	}
+
+	if b.ArchiveURL != "" {
+		remote.SetArchiveRoot(b.ArchiveURL)
+	}
+
+	remote.Name = b.Name
+	remote.DownloadUdebs = b.DownloadUdebs
+	remote.DownloadSources = b.DownloadSources
+	remote.SkipComponentCheck = b.SkipComponentCheck
+	remote.FilterWithDeps = b.FilterWithDeps
+	remote.Filter = b.Filter
+	remote.Architectures = b.Architectures
+	remote.Components = b.Components
+
+	verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
+	if err != nil {
+		c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+		return
+	}
+
+	resources := []string{string(remote.Key())}
+	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+
+		downloader := context.NewDownloader(out)
+		err := remote.Fetch(downloader, verifier)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if !b.ForceUpdate {
+			err = remote.CheckLock()
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		if remote.Filter != "" {
+			var filterQuery deb.PackageQuery
+
+			filterQuery, err = query.Parse(remote.Filter)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+
+			_, _, err = remote.ApplyFilter(context.DependencyOptions(), filterQuery, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		defer func() {
+			// on any interruption, unlock the mirror
+			e := context.ReOpenDatabase()
+			if e == nil {
+				remote.MarkAsIdle()
+				collection.Update(remote)
+			}
+		}()
+
+		remote.MarkAsUpdating()
+		err = collection.Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		context.GoContextHandleSignals()
+
+		count := len(queue)
+		taskDetail := struct {
+			TotalDownloadSize         int64
+			RemainingDownloadSize     int64
+			TotalNumberOfPackages     int
+			RemainingNumberOfPackages int
+		}{
+			downloadSize, downloadSize, count, count,
+		}
+		detail.Store(taskDetail)
+
+		downloadQueue := make(chan int)
+		taskFinished := make(chan *deb.PackageDownloadTask)
+
+		var (
+			errors  []string
+			errLock sync.Mutex
+		)
+
+		pushError := func(err error) {
+			errLock.Lock()
+			errors = append(errors, err.Error())
+			errLock.Unlock()
+		}
+
+		go func() {
+			for idx := range queue {
+				select {
+				case downloadQueue <- idx:
+				case <-context.Done():
+					return
+				}
+			}
+
+			close(downloadQueue)
+		}()
+
+		// update of task details need to be done in order
+		go func() {
+			for {
+				task, ok := <-taskFinished
+				if !ok {
+					return
+				}
+
+				taskDetail.RemainingDownloadSize -= task.File.Checksums.Size
+				taskDetail.RemainingNumberOfPackages--
+				detail.Store(taskDetail)
+			}
+		}()
+
+		log.Printf("%s: Spawning background processes...\n", b.Name)
+		var wg sync.WaitGroup
+		for i := 0; i < context.Config().DownloadConcurrency; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case idx, ok := <-downloadQueue:
+						if !ok {
+							return
+						}
+
+						task := &queue[idx]
+
+						var e error
+
+						// provision download location
+						task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						// download file...
+						e = context.Downloader().DownloadWithChecksum(
+							context,
+							remote.PackageURL(task.File.DownloadURL()).String(),
+							task.TempDownPath,
+							&task.File.Checksums,
+							b.IgnoreChecksums)
+						if e != nil {
+							pushError(e)
+							continue
+						}
+
+						task.Done = true
+						taskFinished <- task
+					case <-context.Done():
+						return
+					}
+
+				}
+			}()
+		}
+
+		// Wait for all download goroutines to finish
+		log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
+		wg.Wait()
+		log.Printf("%s: Background processes finished\n", b.Name)
+		close(taskFinished)
+
+		for idx := range queue {
+
+			atask := &queue[idx]
+
+			if !atask.Done {
+				// download not finished yet
+				continue
+			}
+
+			// and import it back to the pool
+			atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
+			}
+
+			// update "attached" files if any
+			for _, additionalAtask := range atask.Additional {
+				additionalAtask.File.PoolPath = atask.File.PoolPath
+				additionalAtask.File.Checksums = atask.File.Checksums
+			}
+		}
+
+		select {
+		case <-context.Done():
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+
+		if len(errors) > 0 {
+			log.Printf("%s: Unable to update because of previous errors\n", b.Name)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+		}
+
+		log.Printf("%s: Finalizing download\n", b.Name)
+		remote.FinalizeDownload(collectionFactory, out)
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		log.Printf("%s: Mirror updated successfully!\n", b.Name)
+		return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
+	})
+}

api/mirror_test.go

@@ -0,0 +1,40 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type MirrorSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&MirrorSuite{})
+
+func (s *MirrorSuite) TestGetMirrors(c *C) {
+	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[]")
+}
+
+func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
+	response, _ := s.HTTPRequest("DELETE", "/api/mirrors/does-not-exist", nil)
+	c.Check(response.Code, Equals, 404)
+	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
+}
+
+func (s *MirrorSuite) TestCreateMirror(c *C) {
+	c.ExpectFailure("Need to mock downloads")
+	body, err := json.Marshal(gin.H{
+		"Name":       "dummy",
+		"ArchiveURL": "foobar",
+	})
+	c.Assert(err, IsNil)
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Equals, "")
+}

api/packages.go

@@ -6,7 +6,8 @@ import (
 
 // GET /api/packages/:key
 func apiPackagesShow(c *gin.Context) {
-	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
+	collectionFactory := context.NewCollectionFactory()
+	p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
 		c.AbortWithError(404, err)
 		return

api/publish.go

@@ -2,10 +2,13 @@ package api
 
 import (
 	"fmt"
+	"net/http"
 	"strings"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
@@ -51,22 +54,13 @@ func parseEscapedPath(path string) string {
 
 // GET /publish
 func apiPublishList(c *gin.Context) {
-	localCollection := context.CollectionFactory().LocalRepoCollection()
-	localCollection.RLock()
-	defer localCollection.RUnlock()
-
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.RLock()
-	defer snapshotCollection.RUnlock()
-
-	collection := context.CollectionFactory().PublishedRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
 
 	result := make([]*deb.PublishedRepo, 0, collection.Len())
 
 	err := collection.ForEach(func(repo *deb.PublishedRepo) error {
-		err := collection.LoadComplete(repo, context.CollectionFactory())
+		err := collection.LoadComplete(repo, collectionFactory)
 		if err != nil {
 			return err
 		}
@@ -102,6 +96,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		ButAutomaticUpgrades string
 		ForceOverwrite       bool
 		SkipContents         *bool
+		SkipBz2              *bool
 		Architectures        []string
 		Signing              SigningOptions
 		AcquireByHash        *bool
@@ -123,17 +118,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	}
 
 	var components []string
+	var names []string
 	var sources []interface{}
+	var resources []string
+	collectionFactory := context.NewCollectionFactory()
 
 	if b.SourceKind == "snapshot" {
 		var snapshot *deb.Snapshot
 
-		snapshotCollection := context.CollectionFactory().SnapshotCollection()
-		snapshotCollection.Lock()
-		defer snapshotCollection.Unlock()
+		snapshotCollection := collectionFactory.SnapshotCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
+			names = append(names, source.Name)
 
 			snapshot, err = snapshotCollection.ByName(source.Name)
 			if err != nil {
@@ -141,6 +138,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 				return
 			}
 
+			resources = append(resources, string(snapshot.ResourceKey()))
 			err = snapshotCollection.LoadComplete(snapshot)
 			if err != nil {
 				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
@@ -152,12 +150,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	} else if b.SourceKind == deb.SourceLocalRepo {
 		var localRepo *deb.LocalRepo
 
-		localCollection := context.CollectionFactory().LocalRepoCollection()
-		localCollection.Lock()
-		defer localCollection.Unlock()
+		localCollection := collectionFactory.LocalRepoCollection()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
+			names = append(names, source.Name)
 
 			localRepo, err = localCollection.ByName(source.Name)
 			if err != nil {
@@ -165,6 +162,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 				return
 			}
 
+			resources = append(resources, string(localRepo.Key()))
 			err = localCollection.LoadComplete(localRepo)
 			if err != nil {
 				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
@@ -177,55 +175,68 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().PublishedRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
-
-	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
-		return
-	}
-	if b.Origin != "" {
-		published.Origin = b.Origin
-	}
-	if b.NotAutomatic != "" {
-		published.NotAutomatic = b.NotAutomatic
-	}
-	if b.ButAutomaticUpgrades != "" {
-		published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
-	}
-	published.Label = b.Label
-
-	published.SkipContents = context.Config().SkipContentsPublishing
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	duplicate := collection.CheckDuplicate(published)
-	if duplicate != nil {
-		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
-		c.AbortWithError(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
-		return
-	}
-
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
+	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 
-	err = collection.Add(published)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
-		return
-	}
+	resources = append(resources, string(published.Key()))
+	collection := collectionFactory.PublishedRepoCollection()
 
-	c.JSON(201, published)
+	taskName := fmt.Sprintf("Publish %s: %s", b.SourceKind, strings.Join(names, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskDetail := task.PublishDetail{
+			Detail: detail,
+		}
+		publishOutput := &task.PublishOutput{
+			Progress:      out,
+			PublishDetail: taskDetail,
+		}
+
+		if b.Origin != "" {
+			published.Origin = b.Origin
+		}
+		if b.NotAutomatic != "" {
+			published.NotAutomatic = b.NotAutomatic
+		}
+		if b.ButAutomaticUpgrades != "" {
+			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+		}
+		published.Label = b.Label
+
+		published.SkipContents = context.Config().SkipContentsPublishing
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+
+		published.SkipBz2 = context.Config().SkipBz2Publishing
+		if b.SkipContents != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+
+		duplicate := collection.CheckDuplicate(published)
+		if duplicate != nil {
+			collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
+		}
+
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
+		}
+
+		err = collection.Add(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: published}, nil
+	})
 }
 
 // PUT /publish/:prefix/:distribution
@@ -238,6 +249,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		ForceOverwrite bool
 		Signing        SigningOptions
 		SkipContents   *bool
+		SkipBz2        *bool
 		SkipCleanup    *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
@@ -256,31 +268,23 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
-	// published.LoadComplete would touch local repo collection
-	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.Lock()
-	defer localRepoCollection.Unlock()
-
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.Lock()
-	defer snapshotCollection.Unlock()
-
-	collection := context.CollectionFactory().PublishedRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
 		return
 	}
-	err = collection.LoadComplete(published, context.CollectionFactory())
+	err = collection.LoadComplete(published, collectionFactory)
 	if err != nil {
 		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
 	var updatedComponents []string
+	var updatedSnapshots []string
+	var resources []string
 
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
@@ -299,6 +303,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 				return
 			}
 
+			snapshotCollection := collectionFactory.SnapshotCollection()
 			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err2 != nil {
 				c.AbortWithError(404, err2)
@@ -313,6 +318,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 			published.UpdateSnapshot(snapshotInfo.Component, snapshot)
 			updatedComponents = append(updatedComponents, snapshotInfo.Component)
+			updatedSnapshots = append(updatedSnapshots, snapshot.Name)
 		}
 	} else {
 		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
@@ -323,32 +329,37 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.SkipBz2 != nil {
+		published.SkipBz2 = *b.SkipBz2
+	}
+
 	if b.AcquireByHash != nil {
 		published.AcquireByHash = *b.AcquireByHash
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	err = collection.Update(published)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
-		return
-	}
-
-	if b.SkipCleanup == nil || !*b.SkipCleanup {
-		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
-			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+	resources = append(resources, string(published.Key()))
+	taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
 		if err != nil {
-			c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
-			return
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
-	}
 
-	c.JSON(200, published)
+		err = collection.Update(published)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
+		}
+
+		if b.SkipCleanup == nil || !*b.SkipCleanup {
+			err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+				context.GetPublishedStorage(storage), collectionFactory, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
+	})
 }
 
 // DELETE /publish/:prefix/:distribution
@@ -360,21 +371,25 @@ func apiPublishDrop(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := c.Params.ByName("distribution")
 
-	// published.LoadComplete would touch local repo collection
-	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.Lock()
-	defer localRepoCollection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.PublishedRepoCollection()
 
-	collection := context.CollectionFactory().PublishedRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
-
-	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), force, skipCleanup)
+	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to drop: %s", err))
+		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 
-	c.JSON(200, gin.H{})
+	resources := []string{string(published.Key())}
+
+	taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.Remove(context, storage, prefix, distribution,
+			collectionFactory, out, force, skipCleanup)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
 }

api/repos.go

@@ -2,13 +2,17 @@ package api
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
@@ -17,11 +21,9 @@ import (
 func apiReposList(c *gin.Context) {
 	result := []*deb.LocalRepo{}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
-	defer collection.RUnlock()
-
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(r *deb.LocalRepo) error {
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	collection.ForEach(func(r *deb.LocalRepo) error {
 		result = append(result, r)
 		return nil
 	})
@@ -46,11 +48,9 @@ func apiReposCreate(c *gin.Context) {
 	repo.DefaultComponent = b.DefaultComponent
 	repo.DefaultDistribution = b.DefaultDistribution
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
-
-	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	err := collection.Add(repo)
 	if err != nil {
 		c.AbortWithError(400, err)
 		return
@@ -62,6 +62,7 @@ func apiReposCreate(c *gin.Context) {
 // PUT /api/repos/:name
 func apiReposEdit(c *gin.Context) {
 	var b struct {
+		Name                *string
 		Comment             *string
 		DefaultDistribution *string
 		DefaultComponent    *string
@@ -71,9 +72,8 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -81,6 +81,15 @@ func apiReposEdit(c *gin.Context) {
 		return
 	}
 
+	if b.Name != nil {
+		_, err := collection.ByName(*b.Name)
+		if err == nil {
+			// already exists
+			c.AbortWithError(404, err)
+			return
+		}
+		repo.Name = *b.Name
+	}
 	if b.Comment != nil {
 		repo.Comment = *b.Comment
 	}
@@ -102,9 +111,8 @@ func apiReposEdit(c *gin.Context) {
 
 // GET /api/repos/:name
 func apiReposShow(c *gin.Context) {
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
-	defer collection.RUnlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -118,53 +126,42 @@ func apiReposShow(c *gin.Context) {
 // DELETE /api/repos/:name
 func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	name := c.Params.ByName("name")
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.RLock()
-	defer snapshotCollection.RUnlock()
-
-	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
-	publishedCollection.RLock()
-	defer publishedCollection.RUnlock()
-
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	repo, err := collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
 	}
 
-	published := publishedCollection.ByLocalRepo(repo)
-	if len(published) > 0 {
-		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
-		return
-	}
-
-	if !force {
-		snapshots := snapshotCollection.ByLocalRepoSource(repo)
-		if len(snapshots) > 0 {
-			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
-			return
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Delete repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.ByLocalRepo(repo)
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
 		}
-	}
 
-	err = collection.Drop(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
+		if !force {
+			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
+			}
+		}
 
-	c.JSON(200, gin.H{})
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+	})
 }
 
 // GET /api/repos/:name/packages
 func apiReposPackagesShow(c *gin.Context) {
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -178,11 +175,11 @@ func apiReposPackagesShow(c *gin.Context) {
 		return
 	}
 
-	showPackages(c, repo.RefList())
+	showPackages(c, repo.RefList(), collectionFactory)
 }
 
 // Handler for both add and delete
-func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p *deb.Package) error) {
+func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error) {
 	var b struct {
 		PackageRefs []string
 	}
@@ -191,9 +188,8 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -207,54 +203,54 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		return
 	}
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
-
-	// verify package refs and build package list
-	for _, ref := range b.PackageRefs {
-		var p *deb.Package
-
-		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
+	resources := []string{string(repo.Key())}
+	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Loading packages...\n")
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
 		if err != nil {
-			if err == database.ErrNotFound {
-				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
-			} else {
-				c.AbortWithError(500, err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			var p *deb.Package
+
+			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
+				}
+
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = cb(list, p, out)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
-			return
 		}
-		err = cb(list, p)
+
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
-			c.AbortWithError(400, err)
-			return
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
-	}
-
-	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
-		return
-	}
-
-	c.JSON(200, repo)
-
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
 }
 
 // POST /repos/:name/packages
 func apiReposPackagesAdd(c *gin.Context) {
-	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+	apiReposPackagesAddDelete(c, "Add packages to repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Adding package %s\n", p.Name)
 		return list.Add(p)
 	})
 }
 
 // DELETE /repos/:name/packages
 func apiReposPackagesDelete(c *gin.Context) {
-	apiReposPackagesAddDelete(c, func(list *deb.PackageList, p *deb.Package) error {
+	apiReposPackagesAddDelete(c, "Delete packages from repo ", func(list *deb.PackageList, p *deb.Package, out aptly.Progress) error {
+		out.Printf("Removing package %s\n", p.Name)
 		list.Remove(p)
 		return nil
 	})
@@ -275,17 +271,18 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
+	dirParam := c.Params.ByName("dir")
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
 		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
@@ -297,75 +294,91 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	verifier := context.GetVerifier()
-
-	var (
-		sources                      []string
-		packageFiles, failedFiles    []string
-		otherFiles                   []string
-		processedFiles, failedFiles2 []string
-		reporter                     = &aptly.RecordingResultReporter{
-			Warnings:     []string{},
-			AddedLines:   []string{},
-			RemovedLines: []string{},
-		}
-		list *deb.PackageList
-	)
-
+	var taskName string
+	var sources []string
 	if fileParam == "" {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+		taskName = fmt.Sprintf("Add packages from dir %s to repo %s", dirParam, name)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
 	} else {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
+		taskName = fmt.Sprintf("Add package %s from dir %s to repo %s", fileParam, dirParam, name)
 	}
 
-	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+	resources := []string{string(repo.Key())}
+	resources = append(resources, sources...)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		verifier := context.GetVerifier()
 
-	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
-		return
-	}
-
-	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
-	failedFiles = append(failedFiles, failedFiles2...)
-
-	processedFiles = append(processedFiles, otherFiles...)
-
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
-		return
-	}
-
-	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
-		return
-	}
-
-	if !noRemove {
-		processedFiles = utils.StrSliceDeduplicate(processedFiles)
-
-		for _, file := range processedFiles {
-			err := os.Remove(file)
-			if err != nil {
-				reporter.Warning("unable to remove file %s: %s", file, err)
+		var (
+			packageFiles, failedFiles    []string
+			otherFiles                   []string
+			processedFiles, failedFiles2 []string
+			reporter                     = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
 			}
+			list *deb.PackageList
+		)
+
+		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
 		}
 
-		// atempt to remove dir, if it fails, that's fine: probably it's not empty
-		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
-	}
+		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
+			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+		failedFiles = append(failedFiles, failedFiles2...)
+		processedFiles = append(processedFiles, otherFiles...)
 
-	if failedFiles == nil {
-		failedFiles = []string{}
-	}
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
+		}
 
-	c.JSON(200, gin.H{
-		"Report":      reporter,
-		"FailedFiles": failedFiles,
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
+
+		err = collectionFactory.LocalRepoCollection().Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
+		}
+
+		if !noRemove {
+			processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+			for _, file := range processedFiles {
+				err := os.Remove(file)
+				if err != nil {
+					reporter.Warning("unable to remove file %s: %s", file, err)
+				}
+			}
+
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
 	})
 }
 
@@ -383,62 +396,100 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
 
 	repoTemplateString := c.Params.ByName("name")
+	collectionFactory := context.NewCollectionFactory()
 
 	if !verifyDir(c) {
 		return
 	}
 
+	var sources []string
+	var taskName string
+	dirParam := c.Params.ByName("dir")
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
 		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
-	var (
-		err                       error
-		verifier                  = context.GetVerifier()
-		sources, changesFiles     []string
-		failedFiles, failedFiles2 []string
-		reporter                  = &aptly.RecordingResultReporter{
-			Warnings:     []string{},
-			AddedLines:   []string{},
-			RemovedLines: []string{},
-		}
-	)
-
 	if fileParam == "" {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+		taskName = fmt.Sprintf("Include packages from changes files in dir %s to repo matching template %s", dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam)}
 	} else {
-		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+		taskName = fmt.Sprintf("Include packages from changes file %s from dir %s to repo matching template %s", fileParam, dirParam, repoTemplateString)
+		sources = []string{filepath.Join(context.UploadPath(), dirParam, fileParam)}
 	}
 
-	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.Lock()
-	defer localRepoCollection.Unlock()
-
-	changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
-	_, failedFiles2, err = deb.ImportChangesFiles(
-		changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
-		repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
-		context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
-	failedFiles = append(failedFiles, failedFiles2...)
-
+	repoTemplate, err := template.New("repo").Parse(repoTemplateString)
 	if err != nil {
-		c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
+		c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
 		return
 	}
 
-	if !noRemoveFiles {
-		// atempt to remove dir, if it fails, that's fine: probably it's not empty
-		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
-	}
+	var resources []string
+	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+		resources = append(resources, task.AllLocalReposResourcesKey)
+	} else {
+		// repo template string is simple text so only use resource key of specific repository
+		repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
+		if err != nil {
+			c.AbortWithError(404, err)
+			return
+		}
 
-	if failedFiles == nil {
-		failedFiles = []string{}
+		resources = append(resources, string(repo.Key()))
 	}
+	resources = append(resources, sources...)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		var (
+			err                       error
+			verifier                  = context.GetVerifier()
+			changesFiles              []string
+			failedFiles, failedFiles2 []string
+			reporter                  = &aptly.RecordingResultReporter{
+				Warnings:     []string{},
+				AddedLines:   []string{},
+				RemovedLines: []string{},
+			}
+		)
+
+		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+		_, failedFiles2, err = deb.ImportChangesFiles(
+			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+		failedFiles = append(failedFiles, failedFiles2...)
+
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import changes files: %s", err)
+		}
+
+		if !noRemoveFiles {
+			// atempt to remove dir, if it fails, that's fine: probably it's not empty
+			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+		}
+
+		if failedFiles == nil {
+			failedFiles = []string{}
+		}
+
+		if len(reporter.AddedLines) > 0 {
+			out.Printf("Added: %s\n", strings.Join(reporter.AddedLines, ", "))
+		}
+		if len(reporter.RemovedLines) > 0 {
+			out.Printf("Removed: %s\n", strings.Join(reporter.RemovedLines, ", "))
+		}
+		if len(reporter.Warnings) > 0 {
+			out.Printf("Warnings: %s\n", strings.Join(reporter.Warnings, ", "))
+		}
+		if len(failedFiles) > 0 {
+			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+			"Report":      reporter,
+			"FailedFiles": failedFiles,
+		}}, nil
 
-	c.JSON(200, gin.H{
-		"Report":      reporter,
-		"FailedFiles": failedFiles,
 	})
 }

api/router.go

@@ -5,30 +5,46 @@ import (
 
 	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/gin-gonic/gin"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var context *ctx.AptlyContext
 
+func apiMetricsGet() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		promhttp.Handler().ServeHTTP(c.Writer, c.Request)
+	}
+}
+
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
 	context = c
 
 	router := gin.Default()
+	router.UseRawPath = true
 	router.Use(gin.ErrorLogger())
 
+	if c.Config().EnableMetricsEndpoint {
+		router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
+		router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
+		router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
+		router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
+		router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
+	}
+
 	if context.Flags().Lookup("no-lock").Value.Get().(bool) {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
 		// running, we close the database to free the lock.
-		requests := make(chan dbRequest)
+		dbRequests = make(chan dbRequest)
 
-		go acquireDatabase(requests)
+		go acquireDatabase()
 
 		router.Use(func(c *gin.Context) {
 			var err error
 
 			errCh := make(chan error)
-			requests <- dbRequest{acquiredb, errCh}
+			dbRequests <- dbRequest{acquiredb, errCh}
 
 			err = <-errCh
 			if err != nil {
@@ -37,7 +53,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 			}
 
 			defer func() {
-				requests <- dbRequest{releasedb, errCh}
+				dbRequests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
 					c.AbortWithError(500, err)
@@ -46,14 +62,14 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			c.Next()
 		})
-
-	} else {
-		go cacheFlusher()
 	}
 
 	root := router.Group("/api")
 
 	{
+		if c.Config().EnableMetricsEndpoint {
+			root.GET("/metrics", apiMetricsGet())
+		}
 		root.GET("/version", apiVersion)
 	}
 
@@ -81,6 +97,19 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
 	}
 
+	{
+		root.GET("/mirrors", apiMirrorsList)
+		root.GET("/mirrors/:name", apiMirrorsShow)
+		root.GET("/mirrors/:name/packages", apiMirrorsPackages)
+		root.POST("/mirrors", apiMirrorsCreate)
+		root.PUT("/mirrors/:name", apiMirrorsUpdate)
+		root.DELETE("/mirrors/:name", apiMirrorsDrop)
+	}
+
+	{
+		root.POST("/gpg/key", apiGPGAddKey)
+	}
+
 	{
 		root.GET("/files", apiFilesListDirs)
 		root.POST("/files/:dir", apiFilesUpload)
@@ -114,6 +143,21 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	{
 		root.GET("/graph.:ext", apiGraph)
 	}
+	{
+		root.POST("/db/cleanup", apiDbCleanup)
+	}
+	{
+		root.GET("/tasks", apiTasksList)
+		root.POST("/tasks-clear", apiTasksClear)
+		root.GET("/tasks-wait", apiTasksWait)
+		root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
+		root.GET("/tasks/:id/output", apiTasksOutputShow)
+		root.GET("/tasks/:id/detail", apiTasksDetailShow)
+		root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
+		root.GET("/tasks/:id", apiTasksShow)
+		root.DELETE("/tasks/:id", apiTasksDelete)
+		root.POST("/tasks-dummy", apiTasksDummy)
+	}
 
 	return router
 }

api/snapshot.go

@@ -2,9 +2,12 @@ package api
 
 import (
 	"fmt"
+	"net/http"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 )
 
@@ -12,9 +15,8 @@ import (
 func apiSnapshotsList(c *gin.Context) {
 	SortMethodString := c.Request.URL.Query().Get("sort")
 
-	collection := context.CollectionFactory().SnapshotCollection()
-	collection.RLock()
-	defer collection.RUnlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	if SortMethodString == "" {
 		SortMethodString = "name"
@@ -46,49 +48,46 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().RemoteRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.Lock()
-	defer snapshotCollection.Unlock()
-
-	repo, err = collection.ByName(c.Params.ByName("name"))
+	repo, err = collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
 	}
 
-	err = repo.CheckLock()
-	if err != nil {
-		c.AbortWithError(409, err)
-		return
-	}
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := repo.CheckLock()
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
+		}
 
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
+		err = collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
 
-	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
-	if err != nil {
-		c.AbortWithError(400, err)
-		return
-	}
+		snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
 
-	if b.Description != "" {
-		snapshot.Description = b.Description
-	}
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
 
-	err = snapshotCollection.Add(snapshot)
-	if err != nil {
-		c.AbortWithError(400, err)
-		return
-	}
-
-	c.JSON(201, snapshot)
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
 }
 
 // POST /api/snapshots
@@ -115,9 +114,9 @@ func apiSnapshotsCreate(c *gin.Context) {
 		}
 	}
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.Lock()
-	defer snapshotCollection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	var resources []string
 
 	sources := make([]*deb.Snapshot, len(b.SourceSnapshots))
 
@@ -133,39 +132,36 @@ func apiSnapshotsCreate(c *gin.Context) {
 			c.AbortWithError(500, err)
 			return
 		}
+
+		resources = append(resources, string(sources[i].ResourceKey()))
 	}
 
-	list := deb.NewPackageList()
+	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		list := deb.NewPackageList()
 
-	// verify package refs and build package list
-	for _, ref := range b.PackageRefs {
-		var p *deb.Package
-
-		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
-		if err != nil {
-			if err == database.ErrNotFound {
-				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
-			} else {
-				c.AbortWithError(500, err)
+		// verify package refs and build package list
+		for _, ref := range b.PackageRefs {
+			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
+			if err != nil {
+				if err == database.ErrNotFound {
+					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+				}
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			err = list.Add(p)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
-			return
 		}
-		err = list.Add(p)
+
+		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
-			c.AbortWithError(400, err)
-			return
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
-	}
-
-	snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
-
-	err = snapshotCollection.Add(snapshot)
-	if err != nil {
-		c.AbortWithError(400, err)
-		return
-	}
-
-	c.JSON(201, snapshot)
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
+	})
 }
 
 // POST /api/repos/:name/snapshots
@@ -185,43 +181,41 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.LocalRepoCollection()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.Lock()
-	defer snapshotCollection.Unlock()
-
-	repo, err = collection.ByName(c.Params.ByName("name"))
+	repo, err = collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
 	}
 
-	err = collection.LoadComplete(repo)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
+	// including snapshot resource key
+	resources := []string{string(repo.Key()), "S" + b.Name}
+	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		err := collection.LoadComplete(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
 
-	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
-	if err != nil {
-		c.AbortWithError(400, err)
-		return
-	}
+		snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
 
-	if b.Description != "" {
-		snapshot.Description = b.Description
-	}
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
 
-	err = snapshotCollection.Add(snapshot)
-	if err != nil {
-		c.AbortWithError(400, err)
-		return
-	}
-
-	c.JSON(201, snapshot)
+		err = snapshotCollection.Add(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
+	})
 }
 
 // PUT /api/snapshots/:name
@@ -240,44 +234,44 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
-	collection := context.CollectionFactory().SnapshotCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
+	name := c.Params.ByName("name")
 
-	snapshot, err = collection.ByName(c.Params.ByName("name"))
+	snapshot, err = collection.ByName(name)
 	if err != nil {
 		c.AbortWithError(404, err)
 		return
 	}
 
-	_, err = collection.ByName(b.Name)
-	if err == nil {
-		c.AbortWithError(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
-		return
-	}
+	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		_, err := collection.ByName(b.Name)
+		if err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+		}
 
-	if b.Name != "" {
-		snapshot.Name = b.Name
-	}
+		if b.Name != "" {
+			snapshot.Name = b.Name
+		}
 
-	if b.Description != "" {
-		snapshot.Description = b.Description
-	}
+		if b.Description != "" {
+			snapshot.Description = b.Description
+		}
 
-	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
-
-	c.JSON(200, snapshot)
+		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: snapshot}, nil
+	})
 }
 
 // GET /api/snapshots/:name
 func apiSnapshotsShow(c *gin.Context) {
-	collection := context.CollectionFactory().SnapshotCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -299,13 +293,9 @@ func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
-	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.Lock()
-	defer snapshotCollection.Unlock()
-
-	publishedCollection := context.CollectionFactory().PublishedRepoCollection()
-	publishedCollection.RLock()
-	defer publishedCollection.RUnlock()
+	collectionFactory := context.NewCollectionFactory()
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
@@ -313,37 +303,36 @@ func apiSnapshotsDrop(c *gin.Context) {
 		return
 	}
 
-	published := publishedCollection.BySnapshot(snapshot)
+	resources := []string{string(snapshot.ResourceKey())}
+	taskName := fmt.Sprintf("Delete snapshot %s", name)
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		published := publishedCollection.BySnapshot(snapshot)
 
-	if len(published) > 0 {
-		c.AbortWithError(409, fmt.Errorf("unable to drop: snapshot is published"))
-		return
-	}
-
-	if !force {
-		snapshots := snapshotCollection.BySnapshotSource(snapshot)
-		if len(snapshots) > 0 {
-			c.AbortWithError(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
-			return
+		if len(published) > 0 {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
 		}
-	}
 
-	err = snapshotCollection.Drop(snapshot)
-	if err != nil {
-		c.AbortWithError(500, err)
-		return
-	}
+		if !force {
+			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			if len(snapshots) > 0 {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
+			}
+		}
 
-	c.JSON(200, gin.H{})
+		err = snapshotCollection.Drop(snapshot)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, nil
+	})
 }
 
 // GET /api/snapshots/:name/diff/:withSnapshot
 func apiSnapshotsDiff(c *gin.Context) {
 	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
 
-	collection := context.CollectionFactory().SnapshotCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -370,7 +359,7 @@ func apiSnapshotsDiff(c *gin.Context) {
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		c.AbortWithError(500, err)
 		return
@@ -391,9 +380,8 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 // GET /api/snapshots/:name/packages
 func apiSnapshotsSearchPackages(c *gin.Context) {
-	collection := context.CollectionFactory().SnapshotCollection()
-	collection.Lock()
-	defer collection.Unlock()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
@@ -407,5 +395,5 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 		return
 	}
 
-	showPackages(c, snapshot.RefList())
+	showPackages(c, snapshot.RefList(), collectionFactory)
 }

api/task.go

@@ -0,0 +1,155 @@
+package api
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/task"
+	"github.com/gin-gonic/gin"
+)
+
+// GET /tasks
+func apiTasksList(c *gin.Context) {
+	list := context.TaskList()
+	c.JSON(200, list.GetTasks())
+}
+
+// POST /tasks-clear
+func apiTasksClear(c *gin.Context) {
+	list := context.TaskList()
+	list.Clear()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks-wait
+func apiTasksWait(c *gin.Context) {
+	list := context.TaskList()
+	list.Wait()
+	c.JSON(200, gin.H{})
+}
+
+// GET /tasks/:id/wait
+func apiTasksWaitForTaskByID(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	task, err := list.WaitForTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id
+func apiTasksShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var task task.Task
+	task, err = list.GetTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, task)
+}
+
+// GET /tasks/:id/output
+func apiTasksOutputShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var output string
+	output, err = list.GetTaskOutputByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// GET /tasks/:id/detail
+func apiTasksDetailShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var detail interface{}
+	detail, err = list.GetTaskDetailByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, detail)
+}
+
+// GET /tasks/:id/return_value
+func apiTasksReturnValueShow(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	output, err := list.GetTaskReturnValueByID(int(id))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	c.JSON(200, output)
+}
+
+// DELETE /tasks/:id
+func apiTasksDelete(c *gin.Context) {
+	list := context.TaskList()
+	id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
+	if err != nil {
+		c.AbortWithError(500, err)
+		return
+	}
+
+	var delTask task.Task
+	delTask, err = list.DeleteTaskByID(int(id))
+	if err != nil {
+		c.AbortWithError(400, err)
+		return
+	}
+
+	c.JSON(200, delTask)
+}
+
+// POST /tasks-dummy
+func apiTasksDummy(c *gin.Context) {
+	resources := []string{"dummy"}
+	taskName := fmt.Sprintf("Dummy task")
+	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		out.Printf("Dummy task started\n")
+		detail.Store([]int{1, 2, 3})
+		out.Printf("Dummy task finished\n")
+		return &task.ProcessReturnValue{Code: http.StatusTeapot, Value: []int{1, 2, 3}}, nil
+	})
+}

api/task_test.go

@@ -0,0 +1,75 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/aptly-dev/aptly/task"
+
+	. "gopkg.in/check.v1"
+)
+
+type TaskSuite struct {
+	ApiSuite
+}
+
+var _ = Suite(&TaskSuite{})
+
+func (s *TaskSuite) TestTasksDummy(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy", nil)
+	c.Check(response.Code, Equals, 418)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+}
+
+func (s *TaskSuite) TestTasksDummyAsync(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/wait", t.ID), nil)
+	err = json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/detail", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "[1,2,3]")
+	response, _ = s.HTTPRequest("GET", fmt.Sprintf("/api/tasks/%d/output", t.ID), nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Matches, "\"Dummy task started.*")
+}
+
+func (s *TaskSuite) TestTaskDelete(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	c.Check(response.Body.String(), Equals, "{\"Name\":\"Dummy task\",\"ID\":1,\"State\":0}")
+	// Give the task time to start
+	time.Sleep(time.Second)
+	response, _ = s.HTTPRequest("DELETE", "/api/tasks/1", nil)
+	c.Check(response.Code, Equals, 200)
+}
+
+func (s *TaskSuite) TestTasksClear(c *C) {
+	response, _ := s.HTTPRequest("POST", "/api/tasks-dummy?_async=true", nil)
+	c.Check(response.Code, Equals, 202)
+	var t task.Task
+	err := json.Unmarshal(response.Body.Bytes(), &t)
+	c.Assert(err, IsNil)
+	c.Check(t.Name, Equals, "Dummy task")
+	response, _ = s.HTTPRequest("GET", "/api/tasks-wait", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	var ts []task.Task
+	err = json.Unmarshal(response.Body.Bytes(), &ts)
+	c.Assert(err, IsNil)
+	c.Check(len(ts), Equals, 1)
+	c.Check(ts[0].State, Equals, task.SUCCEEDED)
+	response, _ = s.HTTPRequest("POST", "/api/tasks-clear", nil)
+	c.Check(response.Code, Equals, 200)
+	response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
+	c.Check(response.Code, Equals, 200)
+	c.Check(response.Body.String(), Equals, "null")
+}

aptly/interfaces.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"os"
 
+	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/utils"
 )
 
@@ -96,6 +97,36 @@ type PublishedStorageProvider interface {
 	GetPublishedStorage(name string) PublishedStorage
 }
 
+// BarType used to differentiate between different progress bars
+type BarType int
+
+const (
+	// BarGeneralBuildPackageList identifies bar for building package list
+	BarGeneralBuildPackageList BarType = iota
+	// BarGeneralVerifyDependencies identifies bar for verifying dependencies
+	BarGeneralVerifyDependencies
+	// BarGeneralBuildFileList identifies bar for building file list
+	BarGeneralBuildFileList
+	// BarCleanupBuildList identifies bar for building list to cleanup
+	BarCleanupBuildList
+	// BarCleanupDeleteUnreferencedFiles identifies bar for deleting unreferenced files
+	BarCleanupDeleteUnreferencedFiles
+	// BarMirrorUpdateDownloadIndexes identifies bar for downloading index files
+	BarMirrorUpdateDownloadIndexes
+	// BarMirrorUpdateDownloadPackages identifies bar for downloading packages
+	BarMirrorUpdateDownloadPackages
+	// BarMirrorUpdateBuildPackageList identifies bar for building package list of downloaded files
+	BarMirrorUpdateBuildPackageList
+	// BarMirrorUpdateImportFiles identifies bar for importing package files
+	BarMirrorUpdateImportFiles
+	// BarMirrorUpdateFinalizeDownload identifies bar for finalizing downloads
+	BarMirrorUpdateFinalizeDownload
+	// BarPublishGeneratePackageFiles identifies bar for generating package files to publish
+	BarPublishGeneratePackageFiles
+	// BarPublishFinalizeIndexes identifies bar for finalizing index files
+	BarPublishFinalizeIndexes
+)
+
 // Progress is a progress displaying entity, it allows progress bars & simple prints
 type Progress interface {
 	// Writer interface to support progress bar ticking
@@ -107,7 +138,7 @@ type Progress interface {
 	// Flush returns when all queued messages are sent
 	Flush()
 	// InitBar starts progressbar for count bytes or count items
-	InitBar(count int64, isBytes bool)
+	InitBar(count int64, isBytes bool, barType BarType)
 	// ShutdownBar stops progress bar and hides it
 	ShutdownBar()
 	// AddBar increments progress for progress bar
@@ -127,13 +158,16 @@ type Downloader interface {
 	// Download starts new download task
 	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
 	// GetLength returns size by heading object with url
 	GetLength(ctx context.Context, url string) (int64, error)
 }
 
+// ChecksumStorageProvider creates ChecksumStorage based on DB
+type ChecksumStorageProvider func(db database.ReaderWriter) ChecksumStorage
+
 // ChecksumStorage is stores checksums in some (persistent) storage
 type ChecksumStorage interface {
 	// Get finds checksums in DB by path

azure/azure.go

@@ -0,0 +1,2 @@
+// Package azure handles publishing to Azure Storage
+package azure

azure/azure_test.go

@@ -0,0 +1,12 @@
+package azure
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

azure/public.go

@@ -0,0 +1,375 @@
+package azure
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
+	"github.com/pkg/errors"
+)
+
+// PublishedStorage abstract file system with published files (actually hosted on Azure)
+type PublishedStorage struct {
+	container azblob.ContainerURL
+	prefix    string
+	pathCache map[string]string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+func isEmulatorEndpoint(endpoint string) bool {
+	if h, _, err := net.SplitHostPort(endpoint); err == nil {
+		endpoint = h
+	}
+	if endpoint == "localhost" {
+		return true
+	}
+	// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
+	// In this case, eliminate the '[' and ']' in the URL.
+	// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
+	if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
+		endpoint = endpoint[1 : len(endpoint)-1]
+	}
+	return net.ParseIP(endpoint) != nil
+}
+
+// NewPublishedStorage creates published storage from Azure storage credentials
+func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	if err != nil {
+		return nil, err
+	}
+
+	if endpoint == "" {
+		endpoint = "blob.core.windows.net"
+	}
+
+	var url *url.URL
+	if isEmulatorEndpoint(endpoint) {
+		url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
+	} else {
+		url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
+	result := &PublishedStorage{
+		container: containerURL,
+		prefix:    prefix,
+	}
+
+	return result, nil
+}
+
+// String
+func (storage *PublishedStorage) String() string {
+	return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	// no op for Azure
+	return nil
+}
+
+// PutFile puts file into published storage at specified path
+func (storage *PublishedStorage) PutFile(path string, sourceFilename string) error {
+	var (
+		source *os.File
+		err    error
+	)
+
+	sourceMD5, err := utils.MD5ChecksumForFile(sourceFilename)
+	if err != nil {
+		return err
+	}
+
+	source, err = os.Open(sourceFilename)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(path, source, sourceMD5)
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
+	}
+
+	return err
+}
+
+// putFile uploads file-like object to
+func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
+	path = filepath.Join(storage.prefix, path)
+
+	blob := storage.container.NewBlockBlobURL(path)
+
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
+	}
+	if len(sourceMD5) > 0 {
+		decodedMD5, err := hex.DecodeString(sourceMD5)
+		if err != nil {
+			return err
+		}
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
+		}
+	}
+
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob,
+		uploadOptions,
+	)
+
+	return err
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
+	filelist, err := storage.Filelist(path)
+	if err != nil {
+		return err
+	}
+
+	for _, filename := range filelist {
+		blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+		if err != nil {
+			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
+		}
+	}
+
+	return nil
+}
+
+// Remove removes single file under public path
+func (storage *PublishedStorage) Remove(path string) error {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
+	}
+	return err
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
+
+	relPath := filepath.Join(publishedDirectory, fileName)
+	poolPath := filepath.Join(storage.prefix, relPath)
+
+	if storage.pathCache == nil {
+		paths, md5s, err := storage.internalFilelist("")
+		if err != nil {
+			return fmt.Errorf("error caching paths under prefix: %s", err)
+		}
+
+		storage.pathCache = make(map[string]string, len(paths))
+
+		for i := range paths {
+			storage.pathCache[paths[i]] = md5s[i]
+		}
+	}
+
+	destinationMD5, exists := storage.pathCache[relPath]
+	sourceMD5 := sourceChecksums.MD5
+
+	if exists {
+		if sourceMD5 == "" {
+			return fmt.Errorf("unable to compare object, MD5 checksum missing")
+		}
+
+		if destinationMD5 == sourceMD5 {
+			return nil
+		}
+
+		if !force && destinationMD5 != sourceMD5 {
+			return fmt.Errorf("error putting file to %s: file already exists and is different: %s", poolPath, storage)
+		}
+	}
+
+	source, err := sourcePool.Open(sourcePath)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	err = storage.putFile(relPath, source, sourceMD5)
+	if err == nil {
+		storage.pathCache[relPath] = sourceMD5
+	} else {
+		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
+	}
+
+	return err
+}
+
+func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
+	const delimiter = "/"
+	paths = make([]string, 0, 1024)
+	md5s = make([]string, 0, 1024)
+	prefix = filepath.Join(storage.prefix, prefix)
+	if prefix != "" {
+		prefix += delimiter
+	}
+
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := storage.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1000,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		}
+
+		marker = listBlob.NextMarker
+
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
+		}
+	}
+
+	return paths, md5s, nil
+}
+
+// Filelist returns list of files under prefix
+func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
+	paths, _, err := storage.internalFilelist(prefix)
+	return paths, err
+}
+
+// Internal copy or move implementation
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
+	const leaseDuration = 30
+
+	dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
+	srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
+	}
+	defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
+	srcBlobLeaseID := leaseResp.LeaseID()
+
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
+	if err != nil {
+		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
+	}
+
+	copyStatus := copyResp.CopyStatus()
+	for {
+		if copyStatus == azblob.CopyStatusSuccess {
+			if move {
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
+				return err
+			}
+			return nil
+		} else if copyStatus == azblob.CopyStatusPending {
+			time.Sleep(1 * time.Second)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
+			if err != nil {
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
+			}
+			copyStatus = blobPropsResp.CopyStatus()
+
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
+			if err != nil {
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
+			}
+		} else {
+			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
+		}
+	}
+}
+
+// RenameFile renames (moves) file
+func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
+	return storage.internalCopyOrMoveBlob(oldName, newName, nil, true /* move */)
+}
+
+// SymLink creates a copy of src file and adds link information as meta data
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
+}
+
+// HardLink using symlink functionality as hard links do not exist
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return storage.SymLink(src, dst)
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		storageError, ok := err.(azblob.StorageError)
+		if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
+			return false, nil
+		}
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
+	}
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+}
+
+// ReadLink returns the symbolic link pointed to by path.
+// This simply reads text file created with SymLink
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	if err != nil {
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
+	}
+	return resp.NewMetadata()["SymLink"], nil
+}

azure/public_test.go

@@ -0,0 +1,371 @@
+package azure
+
+import (
+	"context"
+	"crypto/md5"
+	"crypto/rand"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+	. "gopkg.in/check.v1"
+)
+
+type PublishedStorageSuite struct {
+	accountName, accountKey, endpoint string
+	storage, prefixedStorage          *PublishedStorage
+}
+
+var _ = Suite(&PublishedStorageSuite{})
+
+const testContainerPrefix = "aptlytest-"
+
+func randContainer() string {
+	return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+	if n <= 0 {
+		panic("negative number")
+	}
+	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+	var bytes = make([]byte, n)
+	rand.Read(bytes)
+	for i, b := range bytes {
+		bytes[i] = alphanum[b%byte(len(alphanum))]
+	}
+	return string(bytes)
+}
+
+func (s *PublishedStorageSuite) SetUpSuite(c *C) {
+	s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
+	if s.accountName == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCOUNT not set.")
+	}
+	s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
+	if s.accountKey == "" {
+		println("Please set the the following two environment variables to run the Azure storage tests.")
+		println("  1. AZURE_STORAGE_ACCOUNT")
+		println("  2. AZURE_STORAGE_ACCESS_KEY")
+		c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
+	}
+	s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
+}
+
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	container := randContainer()
+	prefix := "lala"
+
+	var err error
+
+	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
+	c.Assert(err, IsNil)
+	cnt := s.storage.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
+	c.Assert(err, IsNil)
+
+	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TearDownTest(c *C) {
+	cnt := s.storage.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
+	blob := s.storage.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, IsNil)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := ioutil.ReadAll(body)
+	c.Assert(err, IsNil)
+	return data
+}
+
+func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
+	_, err := s.storage.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
+	c.Assert(err, NotNil)
+	storageError, ok := err.(azblob.StorageError)
+	c.Assert(ok, Equals, true)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
+}
+
+func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
+	hash := md5.Sum(data)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
+	c.Assert(err, IsNil)
+}
+
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+
+	err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
+}
+
+func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
+	content := []byte("Welcome to Azure!")
+	filename := "a/b+c.txt"
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, filename), DeepEquals, content)
+	s.AssertNoFile(c, "a/b c.txt")
+}
+
+func (s *PublishedStorageSuite) TestFilelist(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "test/a", "test/b", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestFilelistPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "test/a 1", "test/a+1", "testa"})
+
+	list, err = s.storage.Filelist("test")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a 1", "a+1"})
+
+	list, err = s.storage.Filelist("test2")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	list, err = s.prefixedStorage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a b", "a+b", "c"})
+}
+
+func (s *PublishedStorageSuite) TestRemove(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+
+	s.PutFile(c, "lala/xyz", []byte("test"))
+
+	err = s.prefixedStorage.Remove("xyz")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "lala/xyz")
+}
+
+func (s *PublishedStorageSuite) TestRemovePlus(c *C) {
+	s.PutFile(c, "a/b+c", []byte("test"))
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.Remove("a/b+c")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b+c")
+	s.AssertNoFile(c, "a/b c")
+
+	err = s.storage.Remove("a/b")
+	c.Check(err, IsNil)
+
+	s.AssertNoFile(c, "a/b")
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
+	paths := []string{"a", "b", "c", "testa", "test/a+1", "test/a 1", "lala/a+b", "lala/a b", "lala/c"}
+	for _, path := range paths {
+		s.PutFile(c, path, []byte("test"))
+	}
+
+	err := s.storage.RemoveDirs("test", nil)
+	c.Check(err, IsNil)
+
+	list, err := s.storage.Filelist("")
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a b", "lala/a+b", "lala/c", "testa"})
+}
+
+func (s *PublishedStorageSuite) TestRenameFile(c *C) {
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	err = s.storage.RenameFile("source.txt", "dest.txt")
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dest.txt"), DeepEquals, []byte("Welcome to Azure!"))
+
+	exists, err := s.storage.FileExists("source.txt")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
+	root := c.MkDir()
+	pool := files.NewPackagePool(root, false)
+	cs := files.NewMockChecksumStorage()
+
+	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	c.Assert(err, IsNil)
+	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
+
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
+	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
+	c.Assert(err, IsNil)
+	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
+	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
+
+	// first link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// duplicate link from pool
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different.*")
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with conflict and force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
+
+	// for prefixed storage:
+	// first link from pool
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
+	c.Check(err, IsNil)
+
+	// 2nd link from pool, providing wrong path for source file
+	//
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
+	s.prefixedStorage.pathCache = nil
+	err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	err := s.storage.SymLink("a/b", "a/b.link")
+	c.Check(err, IsNil)
+
+	var link string
+	link, err = s.storage.ReadLink("a/b.link")
+	c.Check(err, IsNil)
+	c.Check(link, Equals, "a/b")
+
+	c.Skip("copy not available in azure test")
+}
+
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	s.PutFile(c, "a/b", []byte("test"))
+
+	exists, err := s.storage.FileExists("a/b")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+
+	exists, _ = s.storage.FileExists("a/b.invalid")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}

cmd/api_serve.go

@@ -58,25 +58,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-
-		var stat os.FileInfo
-		stat, err = os.Stat(file)
-		shouldRemove := true
-
-		if err == nil && stat.Mode()&os.ModeSocket == os.ModeSocket {
-			shouldRemove = false
-		}
-
-		if err != nil && os.IsNotExist(err) {
-			shouldRemove = false
-		}
-
-		if shouldRemove {
-			err = os.Remove(file)
-			if err != nil {
-				fmt.Printf("Warning: error removing file %s: %s\n", file, err)
-			}
-		}
+		os.Remove(file)
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)

cmd/cmd.go

@@ -21,14 +21,14 @@ const (
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
 		return
 	}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}

cmd/db_cleanup.go

@@ -5,6 +5,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
@@ -21,6 +22,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	verbose := context.Flags().Lookup("verbose").Value.Get().(bool)
 	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// collect information about references packages...
 	existingPackageRefs := deb.NewPackageRefList()
@@ -32,12 +34,12 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
 	}
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -59,17 +61,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	context.CollectionFactory().Flush()
+	collectionFactory.Flush()
 
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
 	}
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -92,17 +94,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	context.CollectionFactory().Flush()
+	collectionFactory.Flush()
 
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
 	}
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -122,19 +124,19 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	context.CollectionFactory().Flush()
+	collectionFactory.Flush()
 
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
 	}
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(published *deb.PublishedRepo) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
 		}
 		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 		if e != nil {
 			return e
 		}
@@ -156,11 +158,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	context.CollectionFactory().Flush()
+	collectionFactory.Flush()
 
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
-	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
+	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
 	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
@@ -183,15 +185,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		if !dryRun {
-			db.StartBatch()
+			batch := db.CreateBatch()
 			err = toDelete.ForEach(func(ref []byte) error {
-				return context.CollectionFactory().PackageCollection().DeleteByKey(ref)
+				return collectionFactory.PackageCollection().DeleteByKey(ref, batch)
 			})
 			if err != nil {
-				return err
+				return fmt.Errorf("unable to delete by key: %s", err)
 			}
 
-			err = db.FinishBatch()
+			err = batch.Write()
 			if err != nil {
 				return fmt.Errorf("unable to write to DB: %s", err)
 			}
@@ -200,15 +202,15 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	context.CollectionFactory().Flush()
+	collectionFactory.Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)
 	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())
-	context.Progress().InitBar(int64(existingPackageRefs.Len()), false)
+	context.Progress().InitBar(int64(existingPackageRefs.Len()), false, aptly.BarCleanupBuildList)
 
 	err = existingPackageRefs.ForEach(func(key []byte) error {
-		pkg, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
+		pkg, err2 := collectionFactory.PackageCollection().ByKey(key)
 		if err2 != nil {
 			tail := ""
 			if verbose {
@@ -259,7 +261,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 
 		if !dryRun {
-			context.Progress().InitBar(int64(len(filesToDelete)), false)
+			context.Progress().InitBar(int64(len(filesToDelete)), false, aptly.BarCleanupDeleteUnreferencedFiles)
 
 			var size, totalSize int64
 			for _, file := range filesToDelete {

cmd/db_recover.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"github.com/aptly-dev/aptly/database"
 	"github.com/smira/commander"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
@@ -15,7 +16,7 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = database.RecoverDB(context.DBPath())
+	err = goleveldb.RecoverDB(context.DBPath())
 
 	return err
 }

cmd/graph.go

@@ -28,8 +28,8 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	layout := context.Flags().Lookup("layout").Value.String()
 
 	fmt.Printf("Generating graph...\n")
-	graph, err := deb.BuildGraph(context.CollectionFactory(), layout)
-
+	collectionFactory := context.NewCollectionFactory()
+	graph, err := deb.BuildGraph(collectionFactory, layout)
 	if err != nil {
 		return err
 	}

cmd/mirror_create.go

@@ -64,7 +64,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to fetch mirror: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Add(repo)
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -102,6 +103,7 @@ Example:
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("force-components", false, "(only with component list) skip check that requested components are listed in Release file")
 	cmd.Flag.Bool("force-architectures", false, "(only with architecture list) skip check that requested architectures are listed in Release file")
+	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd

cmd/mirror_drop.go

@@ -15,8 +15,9 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
@@ -28,7 +29,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByRemoteRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
@@ -40,7 +41,7 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Drop(repo)
+	err = collectionFactory.RemoteRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/mirror_edit.go

@@ -16,7 +16,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -74,7 +75,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/mirror_list.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 
@@ -9,17 +10,29 @@ import (
 )
 
 func aptlyMirrorList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().RemoteRepoCollection().Len())
+	if jsonFlag {
+		return aptlyMirrorListJSON(cmd, args)
+	}
+
+	return aptlyMirrorListTxt(cmd, args)
+}
+
+func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
+	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
+
+	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -52,6 +65,32 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdMirrorList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorList,
@@ -66,6 +105,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/mirror_rename.go

@@ -20,7 +20,8 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
@@ -30,13 +31,13 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
+	_, err = collectionFactory.RemoteRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: mirror %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/mirror_show.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/aptly-dev/aptly/deb"
@@ -11,20 +13,32 @@ import (
 )
 
 func aptlyMirrorShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyMirrorShowJSON(cmd, args)
+	}
+
+	return aptlyMirrorShowTxt(cmd, args)
+}
+
+func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -72,13 +86,56 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList())
+			ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
 	return err
 }
 
+func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				repo.Packages = append(repo.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(repo.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdMirrorShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorShow,
@@ -94,6 +151,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
 
 	return cmd

cmd/mirror_update.go

@@ -22,12 +22,13 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -41,7 +42,6 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
-	maxTries := context.Flags().Lookup("max-tries").Value.Get().(int)
 
 	verifier, err := getVerifier(context.Flags())
 	if err != nil {
@@ -54,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -84,8 +84,8 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(),
-		context.CollectionFactory().ChecksumCollection(), skipExistingPackages)
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -96,12 +96,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+			collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
 	repo.MarkAsUpdating()
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -117,7 +117,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
 
 	// Download from the queue
-	context.Progress().InitBar(downloadSize, true)
+	context.Progress().InitBar(downloadSize, true, aptly.BarMirrorUpdateDownloadPackages)
 
 	downloadQueue := make(chan int)
 
@@ -173,8 +173,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
-						ignoreMismatch,
-						maxTries)
+						ignoreMismatch)
 					if e != nil {
 						pushError(e)
 						continue
@@ -199,7 +198,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	// Import downloaded files
-	context.Progress().InitBar(int64(len(queue)), false)
+	context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
 
 	for idx := range queue {
 		context.Progress().AddBar(1)
@@ -212,7 +211,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 
 		// and import it back to the pool
-		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
 		if err != nil {
 			return fmt.Errorf("unable to import file: %s", err)
 		}
@@ -236,8 +235,8 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
-	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
-	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	repo.FinalizeDownload(collectionFactory, context.Progress())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -268,6 +267,7 @@ Example:
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
+	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 

cmd/package_search.go

@@ -29,7 +29,8 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 		q = &deb.MatchAllQuery{}
 	}
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 	if result.Len() == 0 {
 		return fmt.Errorf("no results")
 	}

cmd/package_show.go

@@ -12,9 +12,9 @@ import (
 	"github.com/smira/flag"
 )
 
-func printReferencesTo(p *deb.Package) (err error) {
-	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
+	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -29,8 +29,8 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -45,8 +45,8 @@ func printReferencesTo(p *deb.Package) (err error) {
 		return err
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -76,7 +76,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 	w := bufio.NewWriter(os.Stdout)
 
-	result := q.Query(context.CollectionFactory().PackageCollection())
+	collectionFactory := context.NewCollectionFactory()
+	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
 		p.Stanza().WriteTo(w, p.IsSource, false, false)
@@ -104,7 +105,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p)
+			printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 

cmd/publish_drop.go

@@ -23,8 +23,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 
 	storage, prefix := deb.ParsePrefix(param)
 
-	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(),
+	collectionFactory := context.NewCollectionFactory()
+	err = collectionFactory.PublishedRepoCollection().Remove(context, storage, prefix, distribution,
+		collectionFactory, context.Progress(),
 		context.Flags().Lookup("force-drop").Value.Get().(bool),
 		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {

cmd/publish_list.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
 	"sort"
 
 	"github.com/aptly-dev/aptly/deb"
@@ -9,19 +11,33 @@ import (
 )
 
 func aptlyPublishList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishListJSON(cmd, args)
+	}
+
+	return aptlyPublishListTxt(cmd, args)
+}
+
+func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
 			return e
 		}
 
@@ -61,6 +77,42 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+		if e != nil {
+			fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
+				repo.StoragePrefix(), repo.Distribution, repo.Components())
+			return e
+		}
+
+		repos = append(repos, repo)
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].GetPath() < repos[j].GetPath()
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdPublishList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishList,
@@ -75,6 +127,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/publish_repo.go

@@ -37,15 +37,17 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 

cmd/publish_show.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -9,12 +10,23 @@ import (
 )
 
 func aptlyPublishShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) < 1 || len(args) > 2 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyPublishShowJSON(cmd, args)
+	}
+
+	return aptlyPublishShowTxt(cmd, args)
+}
+
+func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	distribution := args[0]
 	param := "."
 
@@ -24,7 +36,8 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 
 	storage, prefix := deb.ParsePrefix(param)
 
-	repo, err := context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -42,13 +55,13 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	for component, sourceID := range repo.Sources {
 		var name string
 		if repo.SourceKind == deb.SourceSnapshot {
-			source, e := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			source, e := collectionFactory.SnapshotCollection().ByUUID(sourceID)
 			if e != nil {
 				continue
 			}
 			name = source.Name
 		} else if repo.SourceKind == deb.SourceLocalRepo {
-			source, e := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			source, e := collectionFactory.LocalRepoCollection().ByUUID(sourceID)
 			if e != nil {
 				continue
 			}
@@ -63,6 +76,36 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	distribution := args[0]
+	param := "."
+
+	if len(args) == 2 {
+		param = args[1]
+	}
+
+	storage, prefix := deb.ParsePrefix(param)
+
+	repo, err := context.NewCollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
+	if err != nil {
+		return err
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(repo, "", "  "); err == nil {
+		fmt.Println(string(output))
+	}
+
+	return err
+}
+
 func makeCmdPublishShow() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyPublishShow,
@@ -77,5 +120,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
+
 	return cmd
 }

cmd/publish_snapshot.go

@@ -15,6 +15,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	var err error
 
 	components := strings.Split(context.Flags().Lookup("component").Value.String(), ",")
+	collectionFactory := context.NewCollectionFactory()
 
 	if len(args) < len(components) || len(args) > len(components)+1 {
 		cmd.Usage()
@@ -43,12 +44,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+			snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -79,12 +80,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		)
 
 		for _, name := range args {
-			localRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+			localRepo, err = collectionFactory.LocalRepoCollection().ByName(name)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = context.CollectionFactory().LocalRepoCollection().LoadComplete(localRepo)
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -116,7 +117,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
 	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
 
-	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
+	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
@@ -130,6 +131,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.ButAutomaticUpgrades = butAutomaticUpgrades
 	}
 	published.Label = context.Flags().Lookup("label").Value.String()
+	published.Suite = context.Flags().Lookup("suite").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
 
@@ -137,13 +139,18 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	published.SkipBz2 = context.Config().SkipBz2Publishing
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
 	if context.Flags().IsSet("acquire-by-hash") {
 		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
 	}
 
-	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
+	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
+		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -158,12 +165,12 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 			"the same package pool.\n")
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Add(published)
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -221,15 +228,17 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("origin", "", "overwrite origin name to publish")
 	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
+	cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 

cmd/publish_switch.go

@@ -39,7 +39,8 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -48,7 +49,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: not a snapshot publish")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -67,12 +68,12 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to switch: component %s is not in published repository", component)
 		}
 
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(names[i])
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -95,20 +96,24 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -147,11 +152,12 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")

cmd/publish_update.go

@@ -25,7 +25,8 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 
 	var published *deb.PublishedRepo
 
-	published, err = context.CollectionFactory().PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
+	collectionFactory := context.NewCollectionFactory()
+	published, err = collectionFactory.PublishedRepoCollection().ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -34,7 +35,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: not a local repository publish")
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -59,20 +60,24 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
-	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, context.Progress(), forceOverwrite)
+	if context.Flags().IsSet("skip-bz2") {
+		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
+	}
+
+	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().Update(published)
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
 	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
 	if !skipCleanup {
-		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		err = collectionFactory.PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+			context.GetPublishedStorage(storage), collectionFactory, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -106,11 +111,12 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
-	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
-	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passphrase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
+	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 

cmd/repo_add.go

@@ -22,19 +22,20 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	verifier := context.GetVerifier()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -48,8 +49,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
-		context.CollectionFactory().ChecksumCollection())
+		collectionFactory.PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		collectionFactory.ChecksumCollection)
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)
@@ -59,7 +60,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}

cmd/repo_create.go

@@ -27,15 +27,16 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	collectionFactory := context.NewCollectionFactory()
 	if len(args) == 4 {
 		var snapshot *deb.Snapshot
 
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(args[3])
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(args[3])
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
@@ -43,7 +44,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		repo.UpdateRefList(snapshot.RefList())
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Add(repo)
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}

cmd/repo_drop.go

@@ -15,17 +15,18 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().ByLocalRepo(repo)
+	published := collectionFactory.PublishedRepoCollection().ByLocalRepo(repo)
 	if len(published) > 0 {
 		fmt.Printf("Local repo `%s` is published currently:\n", repo.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -37,7 +38,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().ByLocalRepoSource(repo)
+		snapshots := collectionFactory.SnapshotCollection().ByLocalRepoSource(repo)
 
 		if len(snapshots) > 0 {
 			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
@@ -49,7 +50,7 @@ func aptlyRepoDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Drop(repo)
+	err = collectionFactory.LocalRepoCollection().Drop(repo)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/repo_edit.go

@@ -16,12 +16,13 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -52,7 +53,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/repo_include.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
@@ -31,6 +32,13 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
 	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
+	collectionFactory := context.NewCollectionFactory()
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return fmt.Errorf("error parsing -repo template: %s", err)
+	}
 
 	uploaders := (*deb.Uploaders)(nil)
 	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
@@ -54,9 +62,9 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
 	_, failedFiles2, err = deb.ImportChangesFiles(
-		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplateString,
-		context.Progress(), context.CollectionFactory().LocalRepoCollection(), context.CollectionFactory().PackageCollection(),
-		context.PackagePool(), context.CollectionFactory().ChecksumCollection(),
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
+		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
+		context.PackagePool(), collectionFactory.ChecksumCollection,
 		uploaders, query.Parse)
 	failedFiles = append(failedFiles, failedFiles2...)
 

cmd/repo_list.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"sort"
 
@@ -9,21 +10,33 @@ import (
 )
 
 func aptlyRepoList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoListJSON(cmd, args)
+	}
+
+	return aptlyRepoListTxt(cmd, args)
+}
+
+func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 
-	repos := make([]string, context.CollectionFactory().LocalRepoCollection().Len())
+	collectionFactory := context.NewCollectionFactory()
+	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 			if e != nil {
 				return e
 			}
@@ -58,6 +71,36 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
+	i := 0
+	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if e != nil {
+			return e
+		}
+
+		repos[i] = repo
+		i++
+		return nil
+	})
+
+	context.CloseDatabase()
+
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Name < repos[j].Name
+	})
+	if output, e := json.MarshalIndent(repos, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdRepoList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyRepoList,
@@ -72,6 +115,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 
 	return cmd

cmd/repo_move.go

@@ -19,12 +19,13 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	command := cmd.Name()
 
-	dstRepo, err := context.CollectionFactory().LocalRepoCollection().ByName(args[1])
+	collectionFactory := context.NewCollectionFactory()
+	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(dstRepo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
@@ -35,7 +36,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	)
 
 	if command == "copy" || command == "move" { // nolint: goconst
-		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
+		srcRepo, err = collectionFactory.LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -44,7 +45,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(srcRepo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -53,12 +54,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
-		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
+		srcRemoteRepo, err = collectionFactory.RemoteRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(srcRemoteRepo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -74,12 +75,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	srcList, err := deb.NewPackageListFromRefList(srcRefList, context.CollectionFactory().PackageCollection(), context.Progress())
+	srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -151,7 +152,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	} else {
 		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(dstRepo)
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
@@ -159,7 +160,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}

cmd/repo_remove.go

@@ -18,19 +18,20 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
 	context.Progress().Printf("Loading packages...\n")
 
-	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -60,7 +61,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	} else {
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}

cmd/repo_rename.go

@@ -20,18 +20,19 @@ func aptlyRepoRename(cmd *commander.Command, args []string) error {
 
 	oldName, newName := args[0], args[1]
 
-	repo, err = context.CollectionFactory().LocalRepoCollection().ByName(oldName)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err = collectionFactory.LocalRepoCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().LocalRepoCollection().ByName(newName)
+	_, err = collectionFactory.LocalRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: local repo %s already exists", newName)
 	}
 
 	repo.Name = newName
-	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/repo_show.go

@@ -1,27 +1,42 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
 func aptlyRepoShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlyRepoShowJSON(cmd, args)
+	}
+
+	return aptlyRepoShowTxt(cmd, args)
+}
+
+func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	name := args[0]
 
-	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+	collectionFactory := context.NewCollectionFactory()
+	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -37,7 +52,49 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList())
+		ListPackagesRefList(repo.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include packages if requested
+	packageList := []string{}
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err == nil {
+				packageList = list.FullNames()
+			}
+		}
+
+		sort.Strings(packageList)
+	}
+
+	// merge the repo object with the package list
+	var output []byte
+	if output, err = json.MarshalIndent(struct {
+		*deb.LocalRepo
+		Packages []string
+	}{repo, packageList}, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -57,6 +114,7 @@ ex:
 		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/serve.go

@@ -34,7 +34,8 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		return err
 	}
 
-	if context.CollectionFactory().PublishedRepoCollection().Len() == 0 {
+	collectionFactory := context.NewCollectionFactory()
+	if collectionFactory.PublishedRepoCollection().Len() == 0 {
 		fmt.Printf("No published repositories, unable to serve.\n")
 		return nil
 	}
@@ -56,11 +57,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 
 	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
 
-	sources := make(sort.StringSlice, 0, context.CollectionFactory().PublishedRepoCollection().Len())
-	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
+	sources := make(sort.StringSlice, 0, collectionFactory.PublishedRepoCollection().Len())
+	published := make(map[string]*deb.PublishedRepo, collectionFactory.PublishedRepoCollection().Len())
 
-	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+	err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 		if e != nil {
 			return e
 		}

cmd/snapshot_create.go

@@ -13,13 +13,14 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
+	collectionFactory := context.NewCollectionFactory()
 	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
 		repoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(repoName)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(repoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -29,7 +30,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -44,12 +45,12 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 
 		localRepoName, snapshotName := args[3], args[0]
 
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(localRepoName)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(localRepoName)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -70,7 +71,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Add(snapshot)
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}

cmd/snapshot_diff.go

@@ -15,31 +15,32 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 	}
 
 	onlyMatching := context.Flags().Lookup("only-matching").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name-a> snapshot
-	snapshotA, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshotA, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotA)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
 	// Load <name-b> snapshot
-	snapshotB, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	snapshotB, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshotB)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_drop.go

@@ -15,18 +15,19 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 	}
 
 	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
-	published := context.CollectionFactory().PublishedRepoCollection().BySnapshot(snapshot)
+	published := collectionFactory.PublishedRepoCollection().BySnapshot(snapshot)
 
 	if len(published) > 0 {
 		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
 		for _, repo := range published {
-			err = context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+			err = collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
 			if err != nil {
 				return fmt.Errorf("unable to load published: %s", err)
 			}
@@ -38,7 +39,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 
 	force := context.Flags().Lookup("force").Value.Get().(bool)
 	if !force {
-		snapshots := context.CollectionFactory().SnapshotCollection().BySnapshotSource(snapshot)
+		snapshots := collectionFactory.SnapshotCollection().BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
 			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
 			for _, snap := range snapshots {
@@ -49,7 +50,7 @@ func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().Drop(snapshot)
+	err = collectionFactory.SnapshotCollection().Drop(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to drop: %s", err)
 	}

cmd/snapshot_filter.go

@@ -19,21 +19,22 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	}
 
 	withDeps := context.Flags().Lookup("with-deps").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <source> snapshot
-	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	source, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
 	// Convert snapshot to package list
 	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
-	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -75,7 +76,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
 		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
 
-	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}
@@ -97,7 +98,7 @@ as 'package-name' or as package queries.
 
 Example:
 
-    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priority (required)'
 `,
 		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
 	}

cmd/snapshot_list.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 
 	"github.com/aptly-dev/aptly/deb"
@@ -8,16 +9,28 @@ import (
 )
 
 func aptlySnapshotList(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 0 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
+
+	if jsonFlag {
+		return aptlySnapshotListJSON(cmd, args)
+	}
+
+	return aptlySnapshotListTxt(cmd, args)
+}
+
+func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
+	var err error
+
 	raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
 	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
 
-	collection := context.CollectionFactory().SnapshotCollection()
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
 		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
@@ -46,6 +59,29 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
 	return err
 }
 
+func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
+
+	collection := context.NewCollectionFactory().SnapshotCollection()
+
+	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
+	i := 0
+	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		jsonSnapshots[i] = snapshot
+		i++
+		return nil
+	})
+	if output, e := json.MarshalIndent(jsonSnapshots, "", "  "); e == nil {
+		fmt.Println(string(output))
+	} else {
+		err = e
+	}
+
+	return err
+}
+
 func makeCmdSnapshotList() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlySnapshotList,
@@ -60,6 +96,7 @@ Example:
 `,
 	}
 
+	cmd.Flag.Bool("json", false, "display list in JSON format")
 	cmd.Flag.Bool("raw", false, "display list in machine-readable format")
 	cmd.Flag.String("sort", "name", "display list in 'name' or creation 'time' order")
 

cmd/snapshot_merge.go

@@ -15,15 +15,16 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
+	collectionFactory := context.NewCollectionFactory()
 	sources := make([]*deb.Snapshot, len(args)-1)
 
 	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i+1])
+		sources[i], err = collectionFactory.SnapshotCollection().ByName(args[i+1])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(sources[i])
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
@@ -56,7 +57,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_pull.go

@@ -21,25 +21,26 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	noDeps := context.Flags().Lookup("no-deps").Value.Get().(bool)
 	noRemove := context.Flags().Lookup("no-remove").Value.Get().(bool)
 	allMatches := context.Flags().Lookup("all-matches").Value.Get().(bool)
+	collectionFactory := context.NewCollectionFactory()
 
 	// Load <name> snapshot
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(args[0])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
 	// Load <source> snapshot
-	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[1])
+	source, err := collectionFactory.SnapshotCollection().ByName(args[1])
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -49,12 +50,12 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	// Convert snapshot to package list
 	context.Progress().Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	sourcePackageList, err := deb.NewPackageListFromRefList(source.RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -137,7 +138,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = context.CollectionFactory().SnapshotCollection().Add(destination)
+		err = collectionFactory.SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}

cmd/snapshot_rename.go

@@ -19,19 +19,20 @@ func aptlySnapshotRename(cmd *commander.Command, args []string) error {
 	}
 
 	oldName, newName := args[0], args[1]
+	collectionFactory := context.NewCollectionFactory()
 
-	snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(oldName)
+	snapshot, err = collectionFactory.SnapshotCollection().ByName(oldName)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
-	_, err = context.CollectionFactory().SnapshotCollection().ByName(newName)
+	_, err = collectionFactory.SnapshotCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: snapshot %s already exists", newName)
 	}
 
 	snapshot.Name = newName
-	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/snapshot_search.go

@@ -23,17 +23,18 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	name := args[0]
 	command := cmd.Parent.Name()
+	collectionFactory := context.NewCollectionFactory()
 
 	var reflist *deb.PackageRefList
 
 	if command == "snapshot" { // nolint: goconst
 		var snapshot *deb.Snapshot
-		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
+		snapshot, err = collectionFactory.SnapshotCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -41,12 +42,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		reflist = snapshot.RefList()
 	} else if command == "mirror" {
 		var repo *deb.RemoteRepo
-		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -54,12 +55,12 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		reflist = repo.RefList()
 	} else if command == "repo" { // nolint: goconst
 		var repo *deb.LocalRepo
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
+		repo, err = collectionFactory.LocalRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -69,7 +70,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		panic("unknown command")
 	}
 
-	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}

cmd/snapshot_show.go

@@ -1,7 +1,9 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
+	"sort"
 
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
@@ -9,20 +11,31 @@ import (
 )
 
 func aptlySnapshotShow(cmd *commander.Command, args []string) error {
-	var err error
 	if len(args) != 1 {
 		cmd.Usage()
 		return commander.ErrCommandError
 	}
 
-	name := args[0]
+	jsonFlag := cmd.Flag.Lookup("json").Value.Get().(bool)
 
-	snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+	if jsonFlag {
+		return aptlySnapshotShowJSON(cmd, args)
+	}
+
+	return aptlySnapshotShowTxt(cmd, args)
+}
+
+func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
+	var err error
+	name := args[0]
+	collectionFactory := context.NewCollectionFactory()
+
+	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -37,21 +50,21 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 			var name string
 			if snapshot.SourceKind == deb.SourceSnapshot {
 				var source *deb.Snapshot
-				source, err = context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				source, err = collectionFactory.SnapshotCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
 			} else if snapshot.SourceKind == deb.SourceLocalRepo {
 				var source *deb.LocalRepo
-				source, err = context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				source, err = collectionFactory.LocalRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
 			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
 				var source *deb.RemoteRepo
-				source, err = context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				source, err = collectionFactory.RemoteRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
@@ -66,7 +79,78 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(snapshot.RefList())
+		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+	}
+
+	return err
+}
+
+func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
+	var err error
+
+	name := args[0]
+
+	snapshot, err := context.NewCollectionFactory().SnapshotCollection().ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = context.NewCollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	// include the sources
+	if len(snapshot.SourceIDs) > 0 {
+		for _, sourceID := range snapshot.SourceIDs {
+			if snapshot.SourceKind == deb.SourceSnapshot {
+				var source *deb.Snapshot
+				source, err = context.NewCollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.Snapshots = append(snapshot.Snapshots, source)
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
+				var source *deb.LocalRepo
+				source, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.LocalRepos = append(snapshot.LocalRepos, source)
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
+				var source *deb.RemoteRepo
+				source, err = context.NewCollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				if err != nil {
+					continue
+				}
+				snapshot.RemoteRepos = append(snapshot.RemoteRepos, source)
+			}
+		}
+	}
+
+	// include packages if requested
+	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if snapshot.RefList() != nil {
+			var list *deb.PackageList
+			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
+			if err != nil {
+				return fmt.Errorf("unable to get package list: %s", err)
+			}
+
+			list.PrepareIndex()
+			list.ForEachIndexed(func(p *deb.Package) error {
+				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
+				return nil
+			})
+
+			sort.Strings(snapshot.Packages)
+		}
+	}
+
+	var output []byte
+	if output, err = json.MarshalIndent(snapshot, "", "  "); err == nil {
+		fmt.Println(string(output))
 	}
 
 	return err
@@ -87,6 +171,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("json", false, "display record in JSON format")
 	cmd.Flag.Bool("with-packages", false, "show list of packages")
 
 	return cmd

cmd/snapshot_verify.go

@@ -16,13 +16,14 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 	}
 
 	snapshots := make([]*deb.Snapshot, len(args))
+	collectionFactory := context.NewCollectionFactory()
 	for i := range snapshots {
-		snapshots[i], err = context.CollectionFactory().SnapshotCollection().ByName(args[i])
+		snapshots[i], err = collectionFactory.SnapshotCollection().ByName(args[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshots[i])
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
@@ -30,7 +31,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("Loading packages...\n")
 
-	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	packageList, err := deb.NewPackageListFromRefList(snapshots[0].RefList(), collectionFactory.PackageCollection(), context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
@@ -43,7 +44,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 
 	var pL *deb.PackageList
 	for i := 1; i < len(snapshots); i++ {
-		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		pL, err = deb.NewPackageListFromRefList(snapshots[i].RefList(), collectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to load packages: %s", err)
 		}

codecov.yml

@@ -0,0 +1,7 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0%
+        if_ci_failed: error

completion.d/_aptly

@@ -204,6 +204,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                     update)
                         _arguments \
                             "-download-limit=[limit download speed (kB/s)]:kB/s: " \
+                            "-downloader=[downloader to use]:str: " \
                             "-force=[force update mirror even if it is locked by another process]:$bool" \
                             "-ignore-checksums=[ignore checksum mismatches while downloading package files and metadata]:$bool" \
                             "-ignore-signatures=[disable verification of Release file signatures]:$bool" \
@@ -285,6 +286,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                         ;;
                     list)
                         _arguments '1:: :' \
+                            "-json=[display list in JSON format]:$bool" \
                             "-raw=[display list in machine−readable format]:$bool"
                         ;;
                     move)
@@ -300,6 +302,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                         ;;
                     show)
                         _arguments \
+                            "-json=[display record in JSON format]:$bool" \
                             "-with-packages=[show list of packages]:$bool" \
                             "(-)2:repo name:$repos"
                         ;;
@@ -439,10 +442,11 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-force-overwrite=[overwrite files in package pool in case of mismatch]:$bool"
                             "-gpg-key=[GPG key ID to use when signing the release]:gpg key id:$gpg_keys"
                             "-keyring=[GPG keyring to use (instead of default)]:keyring file:_files -g '*.gpg'"
-                            "-passphrase=[GPG passhprase for the key (warning: could be insecure)]:passphrase: "
-                            "-passphrase-file=[GPG passhprase−file for the key (warning: could be insecure)]:passphrase file:_files"
+                            "-passphrase=[GPG passphrase for the key (warning: could be insecure)]:passphrase: "
+                            "-passphrase-file=[GPG passphrase−file for the key (warning: could be insecure)]:passphrase file:_files"
                             "-secret-keyring=[GPG secret keyring to use (instead of default)]:secret-keyring:_files"
                             "-skip-contents=[don’t generate Contents indexes]:$bool"
+                            "-skip-bz2=[don't generate bzipped indexes]:$bool"
                             "-skip-signing=[don’t sign Release files with GPG]:$bool"
                 )
                 local components_options=(
@@ -452,6 +456,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-butautomaticupgrades=[set value for ButAutomaticUpgrades field]:$bool"
                             "-distribution=[distribution name to publish]:distribution:($dists)"
                             "-label=[label to publish]:label: "
+                            "-suite=[suite to publish]:suite: "
                             "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
                             "-origin=[origin name to publish]:origin: "
                             ${components_options[@]}

completion.d/aptly

@@ -216,7 +216,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi
@@ -314,7 +314,11 @@ _aptly()
           ;;
           "list")
             if [[ $numargs -eq 0 ]]; then
-                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-raw -json" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
               return 0
             fi
           ;;
@@ -361,7 +365,7 @@ _aptly()
           "show")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+                COMPREPLY=($(compgen -W "-json -with-packages" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
               fi
@@ -499,7 +503,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
@@ -524,7 +528,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -539,7 +543,7 @@ _aptly()
           "switch")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi

console/progress.go

@@ -69,7 +69,7 @@ func (p *Progress) Flush() {
 }
 
 // InitBar starts progressbar for count bytes or count items
-func (p *Progress) InitBar(count int64, isBytes bool) {
+func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}

console/terminal.go

@@ -3,10 +3,10 @@ package console
 import (
 	"syscall"
 
-	"golang.org/x/crypto/ssh/terminal"
+	"golang.org/x/term"
 )
 
 // RunningOnTerminal checks whether stdout is terminal
 func RunningOnTerminal() bool {
-	return terminal.IsTerminal(syscall.Stdout)
+	return term.IsTerminal(syscall.Stdout)
 }

context/context.go

@@ -15,14 +15,17 @@ import (
 	"time"
 
 	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/azure"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/s3"
 	"github.com/aptly-dev/aptly/swift"
+	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -39,10 +42,10 @@ type AptlyContext struct {
 
 	progress          aptly.Progress
 	downloader        aptly.Downloader
+	taskList          *task.List
 	database          database.Storage
 	packagePool       aptly.PackagePool
 	publishedStorages map[string]aptly.PublishedStorage
-	collectionFactory *deb.CollectionFactory
 	dependencyOptions int
 	architecturesList []string
 	// Debug features
@@ -199,26 +202,66 @@ func (context *AptlyContext) _progress() aptly.Progress {
 	return context.progress
 }
 
+// NewDownloader returns instance of new downloader with given progress
+func (context *AptlyContext) NewDownloader(progress aptly.Progress) aptly.Downloader {
+	context.Lock()
+	defer context.Unlock()
+
+	return context.newDownloader(progress)
+}
+
+// NewDownloader returns instance of new downloader with given progress without locking
+// so it can be used for internal usage.
+func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downloader {
+	var downloadLimit int64
+	limitFlag := context.flags.Lookup("download-limit")
+	if limitFlag != nil {
+		downloadLimit = limitFlag.Value.Get().(int64)
+	}
+	if downloadLimit == 0 {
+		downloadLimit = context.config().DownloadLimit
+	}
+	maxTries := context.config().DownloadRetries + 1
+	maxTriesFlag := context.flags.Lookup("max-tries")
+	if maxTriesFlag != nil {
+		// If flag is defined prefer it to global setting
+		maxTries = maxTriesFlag.Value.Get().(int)
+	}
+	var downloader string = context.config().Downloader
+	downloaderFlag := context.flags.Lookup("downloader")
+	if downloaderFlag != nil {
+		downloader = downloaderFlag.Value.String()
+	}
+
+	if downloader == "grab" {
+		return http.NewGrabDownloader(downloadLimit*1024, maxTries, progress)
+	}
+	return http.NewDownloader(downloadLimit*1024, maxTries, progress)
+}
+
 // Downloader returns instance of current downloader
 func (context *AptlyContext) Downloader() aptly.Downloader {
 	context.Lock()
 	defer context.Unlock()
 
 	if context.downloader == nil {
-		var downloadLimit int64
-		limitFlag := context.flags.Lookup("download-limit")
-		if limitFlag != nil {
-			downloadLimit = limitFlag.Value.Get().(int64)
-		}
-		if downloadLimit == 0 {
-			downloadLimit = context.config().DownloadLimit
-		}
-		context.downloader = http.NewDownloader(downloadLimit*1024, context._progress())
+		context.downloader = context.newDownloader(context._progress())
 	}
 
 	return context.downloader
 }
 
+// TaskList returns instance of current task list
+func (context *AptlyContext) TaskList() *task.List {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.taskList == nil {
+		context.taskList = task.NewList()
+	}
+	return context.taskList
+}
+
 // DBPath builds path to database
 func (context *AptlyContext) DBPath() string {
 	context.Lock()
@@ -244,13 +287,19 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 	if context.database == nil {
 		var err error
 
-		context.database, err = database.NewDB(context.dbPath())
+		context.database, err = goleveldb.NewDB(context.dbPath())
 		if err != nil {
 			return nil, fmt.Errorf("can't instantiate database: %s", err)
 		}
 	}
 
-	tries := context.flags.Lookup("db-open-attempts").Value.Get().(int)
+	var tries int
+	if context.config().DatabaseOpenAttempts == -1 {
+		tries = context.flags.Lookup("db-open-attempts").Value.Get().(int)
+	} else {
+		tries = context.config().DatabaseOpenAttempts
+	}
+
 	const BaseDelay = 10 * time.Second
 	const Jitter = 1 * time.Second
 
@@ -293,20 +342,16 @@ func (context *AptlyContext) ReOpenDatabase() error {
 	return err
 }
 
-// CollectionFactory builds factory producing all kinds of collections
-func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
+// NewCollectionFactory builds factory producing all kinds of collections
+func (context *AptlyContext) NewCollectionFactory() *deb.CollectionFactory {
 	context.Lock()
 	defer context.Unlock()
 
-	if context.collectionFactory == nil {
-		db, err := context._database()
-		if err != nil {
-			Fatal(err)
-		}
-		context.collectionFactory = deb.NewCollectionFactory(db)
+	db, err := context._database()
+	if err != nil {
+		Fatal(err)
 	}
-
-	return context.collectionFactory
+	return deb.NewCollectionFactory(db)
 }
 
 // PackagePool returns instance of PackagePool
@@ -364,6 +409,18 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 			if err != nil {
 				Fatal(err)
 			}
+		} else if strings.HasPrefix(name, "azure:") {
+			params, ok := context.config().AzurePublishRoots[name[6:]]
+			if !ok {
+				Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:]))
+			}
+
+			var err error
+			publishedStorage, err = azure.NewPublishedStorage(
+				params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint)
+			if err != nil {
+				Fatal(err)
+			}
 		} else {
 			Fatal(fmt.Errorf("unknown published storage format: %v", name))
 		}
@@ -468,7 +525,7 @@ func (context *AptlyContext) GoContextHandleSignals() {
 	defer context.Unlock()
 
 	// Catch ^C
-	sigch := make(chan os.Signal)
+	sigch := make(chan os.Signal, 1)
 	signal.Notify(sigch, os.Interrupt)
 
 	var cancel gocontext.CancelFunc

database/database.go

@@ -0,0 +1,75 @@
+// Package database provides KV database for meta-information
+package database
+
+import "errors"
+
+// Errors for Storage
+var (
+	ErrNotFound = errors.New("key not found")
+)
+
+// StorageProcessor is a function to process one single storage entry
+type StorageProcessor func(key []byte, value []byte) error
+
+// Reader provides KV read calls
+type Reader interface {
+	Get(key []byte) ([]byte, error)
+}
+
+// PrefixReader provides prefixed operations
+type PrefixReader interface {
+	HasPrefix(prefix []byte) bool
+	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
+	KeysByPrefix(prefix []byte) [][]byte
+	FetchByPrefix(prefix []byte) [][]byte
+}
+
+// Writer provides KV update/delete calls
+type Writer interface {
+	Put(key []byte, value []byte) error
+	Delete(key []byte) error
+}
+
+// ReaderWriter combines Reader and Writer
+type ReaderWriter interface {
+	Reader
+	Writer
+}
+
+// Storage is an interface to KV storage
+type Storage interface {
+	Reader
+	Writer
+
+	PrefixReader
+
+	CreateBatch() Batch
+	OpenTransaction() (Transaction, error)
+
+	CreateTemporary() (Storage, error)
+
+	Open() error
+	Close() error
+	CompactDB() error
+	Drop() error
+}
+
+// Batch provides a way to pack many writes.
+type Batch interface {
+	Writer
+
+	// Write closes batch and send accumulated writes to the database
+	Write() error
+}
+
+// Transaction provides isolated atomic way to perform updates.
+//
+// Transactions might be expensive.
+// Transaction should always finish with either Discard() or Commit()
+type Transaction interface {
+	Reader
+	Writer
+
+	Commit() error
+	Discard()
+}

database/goleveldb/batch.go

@@ -0,0 +1,34 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+type batch struct {
+	db *leveldb.DB
+	b  *leveldb.Batch
+}
+
+func (b *batch) Put(key, value []byte) error {
+	b.b.Put(key, value)
+
+	return nil
+}
+
+func (b *batch) Delete(key []byte) error {
+	b.b.Delete(key)
+
+	return nil
+}
+
+func (b *batch) Write() error {
+	return b.db.Write(b.b, &opt.WriteOptions{})
+}
+
+// batch should implement database.Batch
+var (
+	_ database.Batch = &batch{}
+)

database/goleveldb/database.go

@@ -0,0 +1,58 @@
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldbstorage "github.com/syndtr/goleveldb/leveldb/storage"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
+	o := &opt.Options{
+		Filter:                 filter.NewBloomFilter(10),
+		OpenFilesCacheCapacity: 256,
+	}
+
+	if throttleCompaction {
+		o.CompactionL0Trigger = 32
+		o.WriteL0PauseTrigger = 96
+		o.WriteL0SlowdownTrigger = 64
+	}
+
+	return leveldb.OpenFile(path, o)
+}
+
+// NewDB creates new instance of DB, but doesn't open it (yet)
+func NewDB(path string) (database.Storage, error) {
+	return &storage{path: path}, nil
+}
+
+// NewOpenDB creates new instance of DB and opens it
+func NewOpenDB(path string) (database.Storage, error) {
+	db, err := NewDB(path)
+	if err != nil {
+		return nil, err
+	}
+
+	return db, db.Open()
+}
+
+// RecoverDB recovers LevelDB database from corruption
+func RecoverDB(path string) error {
+	stor, err := leveldbstorage.OpenFile(path, false)
+	if err != nil {
+		return err
+	}
+
+	db, err := leveldb.Recover(stor, nil)
+	if err != nil {
+		return err
+	}
+
+	db.Close()
+	stor.Close()
+
+	return nil
+}

database/goleveldb/database_test.go

@@ -1,9 +1,12 @@
-package database
+package goleveldb_test
 
 import (
 	"testing"
 
 	. "gopkg.in/check.v1"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // Launch gocheck tests
@@ -13,7 +16,7 @@ func Test(t *testing.T) {
 
 type LevelDBSuite struct {
 	path string
-	db   Storage
+	db   database.Storage
 }
 
 var _ = Suite(&LevelDBSuite{})
@@ -22,7 +25,7 @@ func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
 	s.path = c.MkDir()
-	s.db, err = NewOpenDB(s.path)
+	s.db, err = goleveldb.NewOpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -43,10 +46,10 @@ func (s *LevelDBSuite) TestRecoverDB(c *C) {
 	err = s.db.Close()
 	c.Check(err, IsNil)
 
-	err = RecoverDB(s.path)
+	err = goleveldb.RecoverDB(s.path)
 	c.Check(err, IsNil)
 
-	s.db, err = NewOpenDB(s.path)
+	s.db, err = goleveldb.NewOpenDB(s.path)
 	c.Check(err, IsNil)
 
 	result, err := s.db.Get(key)
@@ -143,11 +146,11 @@ func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
 	c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
-		return ErrNotFound
-	}), Equals, ErrNotFound)
+		return database.ErrNotFound
+	}), Equals, database.ErrNotFound)
 
 	c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
-		return ErrNotFound
+		return database.ErrNotFound
 	}), IsNil)
 
 	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
@@ -176,9 +179,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	err := s.db.Put(key, value)
 	c.Assert(err, IsNil)
 
-	s.db.StartBatch()
-	s.db.Put(key2, value2)
-	s.db.Delete(key)
+	batch := s.db.CreateBatch()
+	batch.Put(key2, value2)
+	batch.Delete(key)
 
 	v, err := s.db.Get(key)
 	c.Check(err, IsNil)
@@ -187,7 +190,7 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	_, err = s.db.Get(key2)
 	c.Check(err, ErrorMatches, "key not found")
 
-	err = s.db.FinishBatch()
+	err = batch.Write()
 	c.Check(err, IsNil)
 
 	v2, err := s.db.Get(key2)
@@ -196,11 +199,6 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 
 	_, err = s.db.Get(key)
 	c.Check(err, ErrorMatches, "key not found")
-
-	c.Check(func() { s.db.FinishBatch() }, Panics, "no batch")
-
-	s.db.StartBatch()
-	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
 }
 
 func (s *LevelDBSuite) TestCompactDB(c *C) {

database/goleveldb/leveldb.go

@@ -0,0 +1,2 @@
+// Package goleveldb implements database interface via goleveldb
+package goleveldb

database/goleveldb/storage.go

@@ -0,0 +1,180 @@
+package goleveldb
+
+import (
+	"bytes"
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+type storage struct {
+	path string
+	db   *leveldb.DB
+}
+
+// CreateTemporary creates new DB of the same type in temp dir
+func (s *storage) CreateTemporary() (database.Storage, error) {
+	tempdir, err := ioutil.TempDir("", "aptly")
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := internalOpen(tempdir, true)
+	if err != nil {
+		return nil, err
+	}
+	return &storage{db: db, path: tempdir}, nil
+}
+
+// Get key value from database
+func (s *storage) Get(key []byte) ([]byte, error) {
+	value, err := s.db.Get(key, nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, database.ErrNotFound
+		}
+		return nil, err
+	}
+
+	return value, nil
+}
+
+// Put saves key to database, if key has the same value in DB already, it is not saved
+func (s *storage) Put(key []byte, value []byte) error {
+	old, err := s.db.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Equal(old, value) {
+			return nil
+		}
+	}
+	return s.db.Put(key, value, nil)
+}
+
+// Delete removes key from DB
+func (s *storage) Delete(key []byte) error {
+	return s.db.Delete(key, nil)
+}
+
+// KeysByPrefix returns all keys that start with prefix
+func (s *storage) KeysByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		key := iterator.Key()
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, keyc)
+	}
+
+	return result
+}
+
+// FetchByPrefix returns all values with keys that start with prefix
+func (s *storage) FetchByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		val := iterator.Value()
+		valc := make([]byte, len(val))
+		copy(valc, val)
+		result = append(result, valc)
+	}
+
+	return result
+}
+
+// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
+func (s *storage) HasPrefix(prefix []byte) bool {
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
+}
+
+// ProcessByPrefix iterates through all entries where key starts with prefix and calls
+// StorageProcessor on key value pair
+func (s *storage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
+	iterator := s.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		err := proc(iterator.Key(), iterator.Value())
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Close finishes DB work
+func (s *storage) Close() error {
+	if s.db == nil {
+		return nil
+	}
+	err := s.db.Close()
+	s.db = nil
+	return err
+}
+
+// Reopen tries to open (re-open) the database
+func (s *storage) Open() error {
+	if s.db != nil {
+		return nil
+	}
+
+	var err error
+	s.db, err = internalOpen(s.path, false)
+	return err
+}
+
+// CreateBatch creates a Batch object
+func (s *storage) CreateBatch() database.Batch {
+	return &batch{
+		db: s.db,
+		b:  &leveldb.Batch{},
+	}
+}
+
+// OpenTransaction creates new transaction.
+func (s *storage) OpenTransaction() (database.Transaction, error) {
+	t, err := s.db.OpenTransaction()
+	if err != nil {
+		return nil, err
+	}
+
+	return &transaction{t: t}, nil
+}
+
+// CompactDB compacts database by merging layers
+func (s *storage) CompactDB() error {
+	return s.db.CompactRange(util.Range{})
+}
+
+// Drop removes all the DB files (DANGEROUS!)
+func (s *storage) Drop() error {
+	if s.db != nil {
+		return errors.New("DB is still open")
+	}
+
+	return os.RemoveAll(s.path)
+}
+
+// Check interface
+var (
+	_ database.Storage = &storage{}
+)

database/goleveldb/transaction.go

@@ -0,0 +1,60 @@
+package goleveldb
+
+import (
+	"bytes"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type transaction struct {
+	t *leveldb.Transaction
+}
+
+// Get implements database.Reader interface.
+func (t *transaction) Get(key []byte) ([]byte, error) {
+	value, err := t.t.Get(key, nil)
+	if err != nil {
+		if err == leveldb.ErrNotFound {
+			return nil, database.ErrNotFound
+		}
+		return nil, err
+	}
+
+	return value, nil
+}
+
+// Put implements database.Writer interface.
+func (t *transaction) Put(key, value []byte) error {
+	old, err := t.t.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Equal(old, value) {
+			return nil
+		}
+	}
+	return t.t.Put(key, value, nil)
+}
+
+// Delete implements database.Writer interface.
+func (t *transaction) Delete(key []byte) error {
+	return t.t.Delete(key, nil)
+}
+
+// Commit finalizes transaction and commits changes to the stable storage.
+func (t *transaction) Commit() error {
+	return t.t.Commit()
+}
+
+// Discard any transaction changes.
+//
+// Discard is safe to call after Commit(), it would be no-op
+func (t *transaction) Discard() {
+	t.t.Discard()
+}
+
+// transaction should implement database.Transaction
+var _ database.Transaction = &transaction{}

database/leveldb.go

@@ -1,267 +0,0 @@
-// Package database provides KV database for meta-information
-package database
-
-import (
-	"bytes"
-	"errors"
-	"io/ioutil"
-	"os"
-
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/filter"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-// Errors for Storage
-var (
-	ErrNotFound = errors.New("key not found")
-)
-
-// StorageProcessor is a function to process one single storage entry
-type StorageProcessor func(key []byte, value []byte) error
-
-// Storage is an interface to KV storage
-type Storage interface {
-	CreateTemporary() (Storage, error)
-	Get(key []byte) ([]byte, error)
-	Put(key []byte, value []byte) error
-	Delete(key []byte) error
-	HasPrefix(prefix []byte) bool
-	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
-	KeysByPrefix(prefix []byte) [][]byte
-	FetchByPrefix(prefix []byte) [][]byte
-	Open() error
-	Close() error
-	StartBatch()
-	FinishBatch() error
-	CompactDB() error
-	Drop() error
-}
-
-type levelDB struct {
-	path  string
-	db    *leveldb.DB
-	batch *leveldb.Batch
-}
-
-// Check interface
-var (
-	_ Storage = &levelDB{}
-)
-
-func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
-	o := &opt.Options{
-		Filter:                 filter.NewBloomFilter(10),
-		OpenFilesCacheCapacity: 256,
-	}
-
-	if throttleCompaction {
-		o.CompactionL0Trigger = 32
-		o.WriteL0PauseTrigger = 96
-		o.WriteL0SlowdownTrigger = 64
-	}
-
-	return leveldb.OpenFile(path, o)
-}
-
-// NewDB creates new instance of DB, but doesn't open it (yet)
-func NewDB(path string) (Storage, error) {
-	return &levelDB{path: path}, nil
-}
-
-// NewOpenDB creates new instance of DB and opens it
-func NewOpenDB(path string) (Storage, error) {
-	db, err := NewDB(path)
-	if err != nil {
-		return nil, err
-	}
-
-	return db, db.Open()
-}
-
-// RecoverDB recovers LevelDB database from corruption
-func RecoverDB(path string) error {
-	stor, err := storage.OpenFile(path, false)
-	if err != nil {
-		return err
-	}
-
-	db, err := leveldb.Recover(stor, nil)
-	if err != nil {
-		return err
-	}
-
-	db.Close()
-	stor.Close()
-
-	return nil
-}
-
-// CreateTemporary creates new DB of the same type in temp dir
-func (l *levelDB) CreateTemporary() (Storage, error) {
-	tempdir, err := ioutil.TempDir("", "aptly")
-	if err != nil {
-		return nil, err
-	}
-
-	db, err := internalOpen(tempdir, true)
-	if err != nil {
-		return nil, err
-	}
-	return &levelDB{db: db, path: tempdir}, nil
-}
-
-// Get key value from database
-func (l *levelDB) Get(key []byte) ([]byte, error) {
-	value, err := l.db.Get(key, nil)
-	if err != nil {
-		if err == leveldb.ErrNotFound {
-			return nil, ErrNotFound
-		}
-		return nil, err
-	}
-
-	return value, nil
-}
-
-// Put saves key to database, if key has the same value in DB already, it is not saved
-func (l *levelDB) Put(key []byte, value []byte) error {
-	if l.batch != nil {
-		l.batch.Put(key, value)
-		return nil
-	}
-	old, err := l.db.Get(key, nil)
-	if err != nil {
-		if err != leveldb.ErrNotFound {
-			return err
-		}
-	} else {
-		if bytes.Equal(old, value) {
-			return nil
-		}
-	}
-	return l.db.Put(key, value, nil)
-}
-
-// Delete removes key from DB
-func (l *levelDB) Delete(key []byte) error {
-	if l.batch != nil {
-		l.batch.Delete(key)
-		return nil
-	}
-	return l.db.Delete(key, nil)
-}
-
-// KeysByPrefix returns all keys that start with prefix
-func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
-	result := make([][]byte, 0, 20)
-
-	iterator := l.db.NewIterator(nil, nil)
-	defer iterator.Release()
-
-	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
-		key := iterator.Key()
-		keyc := make([]byte, len(key))
-		copy(keyc, key)
-		result = append(result, keyc)
-	}
-
-	return result
-}
-
-// FetchByPrefix returns all values with keys that start with prefix
-func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
-	result := make([][]byte, 0, 20)
-
-	iterator := l.db.NewIterator(nil, nil)
-	defer iterator.Release()
-
-	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
-		val := iterator.Value()
-		valc := make([]byte, len(val))
-		copy(valc, val)
-		result = append(result, valc)
-	}
-
-	return result
-}
-
-// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
-func (l *levelDB) HasPrefix(prefix []byte) bool {
-	iterator := l.db.NewIterator(nil, nil)
-	defer iterator.Release()
-	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
-}
-
-// ProcessByPrefix iterates through all entries where key starts with prefix and calls
-// StorageProcessor on key value pair
-func (l *levelDB) ProcessByPrefix(prefix []byte, proc StorageProcessor) error {
-	iterator := l.db.NewIterator(nil, nil)
-	defer iterator.Release()
-
-	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
-		err := proc(iterator.Key(), iterator.Value())
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Close finishes DB work
-func (l *levelDB) Close() error {
-	if l.db == nil {
-		return nil
-	}
-	err := l.db.Close()
-	l.db = nil
-	return err
-}
-
-// Reopen tries to open (re-open) the database
-func (l *levelDB) Open() error {
-	if l.db != nil {
-		return nil
-	}
-
-	var err error
-	l.db, err = internalOpen(l.path, false)
-	return err
-}
-
-// StartBatch starts batch processing of keys
-//
-// All subsequent Get, Put and Delete would work on batch
-func (l *levelDB) StartBatch() {
-	if l.batch != nil {
-		panic("batch already started")
-	}
-	l.batch = new(leveldb.Batch)
-}
-
-// FinishBatch finalizes the batch, saving operations
-func (l *levelDB) FinishBatch() error {
-	if l.batch == nil {
-		panic("no batch")
-	}
-	err := l.db.Write(l.batch, nil)
-	l.batch = nil
-	return err
-}
-
-// CompactDB compacts database by merging layers
-func (l *levelDB) CompactDB() error {
-	return l.db.CompactRange(util.Range{})
-}
-
-// Drop removes all the DB files (DANGEROUS!)
-func (l *levelDB) Drop() error {
-	if l.db != nil {
-		return errors.New("DB is still open")
-	}
-
-	return os.RemoveAll(l.path)
-}

deb/changes.go

@@ -195,11 +195,6 @@ func (c *Changes) PackageQuery() PackageQuery {
 			}
 		}
 
-		ddebQuery = &AndQuery{
-			L: &FieldQuery{Field: "Source", Relation: VersionEqual, Value: c.Source},
-			R: ddebQuery,
-		}
-
 		binaryQuery = &OrQuery{
 			L: binaryQuery,
 			R: ddebQuery,
@@ -293,14 +288,9 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 
 // ImportChangesFiles imports referenced files in changes files into local repository
 func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
-	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
-	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+	verifier pgp.Verifier, repoTemplate *template.Template, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
 
-	var repoTemplate *template.Template
-	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
-	}
 	for _, path := range changesFiles {
 		var changes *Changes
 
@@ -384,7 +374,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		var processedFiles2, failedFiles2 []string
 
 		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
-			packageCollection, reporter, restriction, checksumStorage)
+			packageCollection, reporter, restriction, checksumStorageProvider)
 
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to import package files: %s", err)

deb/changes_test.go

@@ -3,10 +3,12 @@ package deb
 import (
 	"os"
 	"path/filepath"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/console"
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 
@@ -37,7 +39,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
 	c.Assert(err, IsNil)
 
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.localRepoCollection = NewLocalRepoCollection(s.db)
 	s.packageCollection = NewPackageCollection(s.db)
 
@@ -122,13 +124,30 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	processedFiles, failedFiles, err := ImportChangesFiles(
 		append(changesFiles, "testdata/changes/notexistent.changes"),
 		s.Reporter, true, true, false, false, &NullVerifier{},
-		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
 	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
 }
 
+func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	changesFiles, failedFiles := CollectChangesFiles(
+		[]string{"testdata/dbgsym-with-source-version"}, s.Reporter)
+	c.Check(changesFiles, HasLen, 1)
+	c.Check(failedFiles, HasLen, 0)
+
+	_, failedFiles, err := ImportChangesFiles(
+		changesFiles, s.Reporter, true, true, false, true, &NullVerifier{},
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, IsNil)
+}
+
 func (s *ChangesSuite) TestPrepare(c *C) {
 	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
 	c.Assert(err, IsNil)
@@ -148,5 +167,5 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 
 	q := changes.PackageQuery()
 	c.Check(q.String(), Equals,
-		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
+		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym))))))")
 }

deb/checksum_collection.go

@@ -11,12 +11,12 @@ import (
 
 // ChecksumCollection does management of ChecksumInfo in DB
 type ChecksumCollection struct {
-	db          database.Storage
+	db          database.ReaderWriter
 	codecHandle *codec.MsgpackHandle
 }
 
 // NewChecksumCollection creates new ChecksumCollection and binds it to database
-func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+func NewChecksumCollection(db database.ReaderWriter) *ChecksumCollection {
 	return &ChecksumCollection{
 		db:          db,
 		codecHandle: &codec.MsgpackHandle{},

deb/checksum_collection_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/database/goleveldb"
 	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -22,7 +23,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
 		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
 		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 	}
-	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.collection = NewChecksumCollection(s.db)
 }
 

deb/collections.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"sync"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 )
 
@@ -91,10 +92,14 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 }
 
 // ChecksumCollection returns (or creates) new ChecksumCollection
-func (factory *CollectionFactory) ChecksumCollection() *ChecksumCollection {
+func (factory *CollectionFactory) ChecksumCollection(db database.ReaderWriter) aptly.ChecksumStorage {
 	factory.Lock()
 	defer factory.Unlock()
 
+	if db != nil {
+		return NewChecksumCollection(db)
+	}
+
 	if factory.checksums == nil {
 		factory.checksums = NewChecksumCollection(factory.db)
 	}

deb/contents.go

@@ -25,11 +25,11 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
+func (index *ContentsIndex) Push(qualifiedName []byte, contents []string, dbw database.Writer) error {
 	for _, path := range contents {
 		// for performance reasons we only write to leveldb during push.
 		// merging of qualified names per path will be done in WriteTo
-		err := index.db.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
+		err := dbw.Put(append(append(append(index.prefix, []byte(path)...), byte(0)), qualifiedName...), nil)
 		if err != nil {
 			return err
 		}

deb/deb.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/kjk/lzma"
+	"github.com/klauspost/compress/zstd"
 	"github.com/smira/go-xz"
 )
 
@@ -74,6 +75,13 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 				defer unxz.Close()
 				tarInput = unxz
+			case "control.tar.zst":
+				unzstd, err := zstd.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unzstd %s from %s", header.Name, packageFile)
+				}
+				defer unzstd.Close()
+				tarInput = unzstd
 			default:
 				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
 			}
@@ -189,6 +197,13 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 				unlzma := lzma.NewReader(bufReader)
 				defer unlzma.Close()
 				tarInput = unlzma
+			case "data.tar.zst":
+				unzstd, err := zstd.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unzstd %s from %s", header.Name, packageFile)
+				}
+				defer unzstd.Close()
+				tarInput = unzstd
 			default:
 				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
 			}

deb/deb_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 type DebSuite struct {
-	debFile, debFile2, debFileWithXzControl, dscFile, dscFileNoSign string
+	debFile, debFile2, debFileWithXzControl, debFileWithZstdControl, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -21,6 +21,7 @@ func (s *DebSuite) SetUpSuite(c *C) {
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
 	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
 	s.debFileWithXzControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.9.1+dfsg-2+18.04+bionic+build4_amd64.ddeb")
+	s.debFileWithZstdControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.15.2+dfsg-12_amd64.ddeb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -47,6 +48,14 @@ func (s *DebSuite) TestGetControlFileFromDebWithXzControl(c *C) {
 	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
 }
 
+func (s *DebSuite) TestGetControlFileFromDebWithZstdControl(c *C) {
+	// Has control.tar.zstd archive inside.
+	st, err := GetControlFileFromDeb(s.debFileWithZstdControl)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "5.15.2+dfsg-12")
+	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
+}
+
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	verifier := &pgp.GoVerifier{}
 

deb/import.go

@@ -66,11 +66,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 // ImportPackageFiles imports files into local repository
 func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier pgp.Verifier,
 	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery,
-	checksumStorage aptly.ChecksumStorage) (processedFiles []string, failedFiles []string, err error) {
+	checksumStorageProvider aptly.ChecksumStorageProvider) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
 
+	transaction, err := collection.db.OpenTransaction()
+	if err != nil {
+		return nil, nil, err
+	}
+	defer transaction.Discard()
+
+	checksumStorage := checksumStorageProvider(transaction)
+
 	for _, file := range packageFiles {
 		var (
 			stanza Stanza
@@ -193,7 +201,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
-		err = collection.Update(p)
+		err = collection.UpdateInTransaction(p, transaction)
 		if err != nil {
 			reporter.Warning("Unable to save package %s: %s", p, err)
 			failedFiles = append(failedFiles, file)
@@ -219,6 +227,6 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		processedFiles = append(processedFiles, candidateProcessedFiles...)
 	}
 
-	err = nil
+	err = transaction.Commit()
 	return
 }

deb/index_files.go

@@ -22,6 +22,7 @@ type indexFiles struct {
 	suffix           string
 	indexes          map[string]*indexFile
 	acquireByHash    bool
+	skipBz2          bool
 }
 
 type indexFile struct {
@@ -68,7 +69,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	}
 
 	if file.compressable {
-		err = utils.CompressFile(file.tempFile, file.onlyGzip)
+		err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
 		if err != nil {
 			file.tempFile.Close()
 			return fmt.Errorf("unable to compress index file: %s", err)
@@ -80,11 +81,15 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	exts := []string{""}
 	cksumExts := exts
 	if file.compressable {
-		exts = append(exts, ".gz", ".bz2")
-		cksumExts = exts
 		if file.onlyGzip {
 			exts = []string{".gz"}
 			cksumExts = []string{"", ".gz"}
+		} else {
+			exts = append(exts, ".gz")
+			if !file.parent.skipBz2 {
+				exts = append(exts, ".bz2")
+			}
+			cksumExts = exts
 		}
 	}
 
@@ -229,7 +234,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool, skipBz2 bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -239,6 +244,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
 		acquireByHash:    acquireByHash,
+		skipBz2:          skipBz2,
 	}
 }
 
@@ -391,7 +397,7 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 
 func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
-		progress.InitBar(int64(len(files.indexes)), false)
+		progress.InitBar(int64(len(files.indexes)), false, aptly.BarPublishFinalizeIndexes)
 		defer progress.ShutdownBar()
 	}
 

deb/list.go

@@ -99,7 +99,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	result := NewPackageListWithDuplicates(false, reflist.Len())
 
 	if progress != nil {
-		progress.InitBar(int64(reflist.Len()), false)
+		progress.InitBar(int64(reflist.Len()), false, aptly.BarGeneralBuildPackageList)
 	}
 
 	err := reflist.ForEach(func(key []byte) error {
@@ -266,6 +266,19 @@ func (l *PackageList) Strings() []string {
 	return result
 }
 
+// FullNames builds a list of package {name}_{version}_{arch}
+func (l *PackageList) FullNames() []string {
+	result := make([]string, l.Len())
+	i := 0
+
+	for _, p := range l.packages {
+		result[i] = p.GetFullName()
+		i++
+	}
+
+	return result
+}
+
 // depSliceDeduplicate removes dups in slice of Dependencies
 func depSliceDeduplicate(s []Dependency) []Dependency {
 	l := len(s)
@@ -301,7 +314,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
-		progress.InitBar(int64(l.Len())*int64(len(architectures)), false)
+		progress.InitBar(int64(l.Len())*int64(len(architectures)), false, aptly.BarGeneralVerifyDependencies)
 	}
 
 	for _, arch := range architectures {