Merge pull request #847 from smira/fix-repo-name

Fix repo name in release script

.gitignore

@@ -35,3 +35,9 @@ man/aptly.1.ronn
 .goxc.local.json
 
 system/env/
+
+# created by make build for release artifacts
+build/
+
+pgp/keyrings/aptly2*.gpg
+pgp/keyrings/aptly2*.gpg~

.golangci.yml

@@ -0,0 +1,22 @@
+run:
+  tests: false
+
+
+linters:
+  enable-all: false
+  disable-all: true
+  enable:
+   - govet
+   - golint
+   - gofmt
+   - deadcode
+   - goimports
+   - misspell
+   - ineffassign
+   - staticcheck
+   - varcheck
+   - structcheck
+   - maligned
+   - vetshadow
+   - goconst
+   - interfacer

.travis.yml

@@ -10,6 +10,8 @@ addons:
     packages:
     - python-virtualenv
     - graphviz
+    - gnupg2
+    - gpgv2
 
 env:
   global:
@@ -24,12 +26,16 @@ matrix:
       env: RUN_LONG_TESTS=no
   fast_finish: true
   include:
-    - go: 1.8.x
-      env: RUN_LONG_TESTS=no
-    - go: 1.9.x
-      env: RUN_LONG_TESTS=yes
     - go: 1.10.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.11.x
       env: RUN_LONG_TESTS=yes
+    - go: 1.12.x
+      env:
+        - RUN_LONG_TESTS=yes
+        - DEPLOY_BINARIES=yes
+        - APTLY_USER=aptly
+        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
     - go: master
       env: RUN_LONG_TESTS=no
 
@@ -54,3 +60,41 @@ notifications:
     on_success: change  # options: [always|never|change] default: always
     on_failure: always  # options: [always|never|change] default: always
     on_start: false     # default: false
+
+before_deploy:
+  - make release
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
+    file_glob: true
+    file: build/*
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: s3
+    access_key_id:
+      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
+    secret_access_key:
+      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
+    bucket: aptly-nightly
+    skip_cleanup: true
+    acl: public-read
+    local_dir: build
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh nightly
+    skip_cleanup: true
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh release
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -33,3 +33,5 @@ List of contributors, in chronological order:
 * Petr Jediny (https://github.com/pjediny)
 * Maximilian Stein (https://github.com/steinymity)
 * Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)

CONTRIBUTING.md

@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your Fist Code Contribution
+### Your First Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -80,14 +80,14 @@ If you're new to Go, follow [getting started guide](https://golang.org/doc/insta
 initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
 
 Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
-to prepend it or to skip this test if you're security conscious.  
+to prepend it or to skip this test if you're security conscious.
 
 ### Forking and Cloning
 
 As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
 
-    mkdir -p ~/go/src/github.com/smira
+    mkdir -p ~/go/src/github.com/aptly-dev
-    cd ~/go/src/github.com/smira
+    cd ~/go/src/github.com/aptly-dev
     git clone git@github.com:aptly-dev/aptly.git
     cd aptly
 
@@ -184,9 +184,8 @@ Style checks could be run with:
 
     make check
 
-aptly is using [gometalinter](https://github.com/alecthomas/gometalinter) to run style checks on Go code. Configuration
+aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
-for the linter could be found in [linter.json](linter.json) file. Running linters might take considerable amount of time
+for the linter could be found in [.golangci.yml](.golangci.yml) file.
-unfortunately, but usually warning reported by linters hint at real code issues.
 
 Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
 

Gopkg.lock

@@ -2,18 +2,23 @@
 
 
 [[projects]]
+  digest = "1:e8777c437e157121465a08c05304d5847ae70b5683ef4afeb723b3c9e5e3bc67"
   name = "github.com/AlekSi/pointer"
   packages = ["."]
+  pruneopts = ""
   revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
   version = "v1.0.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:218a66570875d243938fff08f17ec03e79b18c138d0845b2428bcb4929ffa4fe"
   name = "github.com/DisposaBoy/JsonConfigReader"
   packages = ["."]
+  pruneopts = ""
   revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
 
 [[projects]]
+  digest = "1:658ab137074ef3d1216e01cbe166eb171a1bd77daeae0b686ab1ae184a9a5ec9"
   name = "github.com/awalterschulze/gographviz"
   packages = [
     ".",
@@ -22,10 +27,12 @@
     "scanner",
     "token"
   ]
+  pruneopts = ""
   revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
   version = "v1.0"
 
 [[projects]]
+  digest = "1:a72e35a51b628d148241720897de644fd6d3cea45d8489dae9d373ced5dfc302"
   name = "github.com/aws/aws-sdk-go"
   packages = [
     "aws",
@@ -56,155 +63,203 @@
     "service/s3",
     "service/sts"
   ]
+  pruneopts = ""
   revision = "a72204b9bf8d48230ee0fe8995613b394c66f2da"
   version = "v1.13.31"
 
 [[projects]]
+  digest = "1:b9922c7da8a89c973758c03bde497017567f785b49bd253402a3587abec1e53d"
   name = "github.com/cheggaaa/pb"
   packages = ["."]
+  pruneopts = ""
   revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
   version = "v1.0.10"
 
 [[projects]]
   branch = "master"
+  digest = "1:1120f960f5c334f0f94bad29eefaf73d52d226893369693686148f66c1993f15"
   name = "github.com/gin-contrib/sse"
   packages = ["."]
+  pruneopts = ""
   revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
 
 [[projects]]
+  digest = "1:348ceb76f2ac958e541e4ba3190484b68df28c38ac9720ed4ef8d36af69ce52e"
   name = "github.com/gin-gonic/gin"
   packages = [
     ".",
     "binding",
     "render"
   ]
+  pruneopts = ""
   revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
   version = "v1.2"
 
 [[projects]]
+  digest = "1:fbed9ba4076145ae05ef7deeb14d49565e112d6c283a8c4304fa3b7be785fa5e"
   name = "github.com/go-ini/ini"
   packages = ["."]
+  pruneopts = ""
   revision = "1730955e3146956d6a087861380f9b4667ed5071"
   version = "v1.26.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:27854310d59099f8dcc61dd8af4a69f0a3597f001154b2fb4d1c41baf2e31ec1"
   name = "github.com/golang/protobuf"
   packages = ["proto"]
+  pruneopts = ""
   revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
 
 [[projects]]
   branch = "master"
+  digest = "1:09307dfb1aa3f49a2bf869dcfa4c6c06ecd3c207221bd1c1a1141f0e51f209eb"
   name = "github.com/golang/snappy"
   packages = ["."]
+  pruneopts = ""
   revision = "553a641470496b2327abcac10b36396bd98e45c9"
 
 [[projects]]
+  digest = "1:6ee50e0ace655f26787c120dbce51b3185c470f475c9e57d0dcada8ebb115004"
   name = "github.com/h2non/filetype"
   packages = ["matchers"]
+  pruneopts = ""
   revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb"
   version = "v1.0.5"
 
 [[projects]]
   branch = "master"
+  digest = "1:64b78d98b8956492576911baf6a1e3499816d4575e485d12792e4abe7d8b6c46"
   name = "github.com/jlaffaye/ftp"
   packages = ["."]
+  pruneopts = ""
   revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
 
 [[projects]]
+  digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e"
   name = "github.com/jmespath/go-jmespath"
   packages = ["."]
+  pruneopts = ""
   revision = "0b12d6b5"
 
 [[projects]]
+  branch = "master"
+  digest = "1:c728183dd470c8bb2d1775edc641a81ec25f5e3804b8b0536e99f71df17a13a6"
+  name = "github.com/kjk/lzma"
+  packages = ["."]
+  pruneopts = ""
+  revision = "3fd93898850d5252457e48c1b3d5e1510597280b"
+
+[[projects]]
+  digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
   name = "github.com/mattn/go-isatty"
   packages = ["."]
+  pruneopts = ""
   revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
   version = "v0.0.3"
 
 [[projects]]
+  digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
   name = "github.com/mattn/go-runewidth"
   packages = ["."]
+  pruneopts = ""
   revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
   version = "v0.0.2"
 
 [[projects]]
+  digest = "1:fb8502ed69803c4cd97b1def119db73869ec2f7e2b408eb9c1b8276df6f05b3e"
   name = "github.com/mattn/go-shellwords"
   packages = ["."]
+  pruneopts = ""
   revision = "005a0944d84452842197c2108bd9168ced206f78"
   version = "v1.0.2"
 
 [[projects]]
   branch = "master"
+  digest = "1:3ba28ef4fbbf8d099c6227b698a6ceae28f8a071e12f06839c12c4c0003b7554"
   name = "github.com/mkrautz/goar"
   packages = ["."]
+  pruneopts = ""
   revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
 
 [[projects]]
   branch = "master"
+  digest = "1:d33ce379780d7c43405b9251289493cabada82f6bf9ab35eea6915d04f6ac8e0"
   name = "github.com/mxk/go-flowrate"
   packages = ["flowrate"]
+  pruneopts = ""
   revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
 
 [[projects]]
   branch = "master"
+  digest = "1:428d8af27f534ed06d783b03d477124796de06aa86402777cd2b494c64278da5"
   name = "github.com/ncw/swift"
   packages = [
     ".",
     "swifttest"
   ]
+  pruneopts = ""
   revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
 
 [[projects]]
+  branch = "master"
+  digest = "1:beeb9206cc21cfeb113066c3dcf4bbb0ba304d73dd441f3244721566f51f44e6"
+  name = "github.com/pborman/uuid"
+  packages = ["."]
+  pruneopts = ""
+  revision = "c65b2f87fee37d1c7854c9164a450713c28d50cd"
+
+[[projects]]
+  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
   name = "github.com/pkg/errors"
   packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  pruneopts = ""
-  version = "v0.8.0"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:c6e71685a14409c3a1fdbecef044ee6447c3887de7c2bb025ee12f9a8a368960"
   name = "github.com/smira/commander"
   packages = ["."]
+  pruneopts = ""
   revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
 
 [[projects]]
   branch = "master"
+  digest = "1:a75eb2e6e718988505f704d651eeb4734f0f0b580f761aa58b4a5b0afe585091"
   name = "github.com/smira/flag"
   packages = ["."]
+  pruneopts = ""
   revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
 
 [[projects]]
   branch = "master"
+  digest = "1:cd3b8057abbb453404cbef569951a7359986eedfc0a887f9bc5efff8d02e1760"
   name = "github.com/smira/go-aws-auth"
   packages = ["."]
-  revision = "0070896e9d7f4f9f2d558532b2d896ce2239992a"
+  pruneopts = ""
+  revision = "8b73995fd8d1d82519c7037ee243a497552cd54d"
 
 [[projects]]
   branch = "master"
+  digest = "1:7dd59b3536a07651baf394466da3b8a20bd3ed9ce1984c7583fdf2412fdcfea9"
   name = "github.com/smira/go-ftp-protocol"
   packages = ["protocol"]
+  pruneopts = ""
   revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
 
 [[projects]]
   branch = "master"
-  name = "github.com/smira/go-uuid"
+  digest = "1:55fe1086100b305a5562e9f1e6825ef97c49e82331cb0f055b2069f7c7ebd469"
-  packages = ["uuid"]
-  revision = "ed3ca8a15a931b141440a7e98e4f716eec255f7d"
-
-[[projects]]
-  branch = "master"
   name = "github.com/smira/go-xz"
   packages = ["."]
+  pruneopts = ""
   revision = "0c531f070014e218b21f3cfca801cc992d52726d"
 
 [[projects]]
   branch = "master"
-  name = "github.com/smira/lzma"
+  digest = "1:175bc8d87d54849b9b9fc6cd473c5830e090fbf3c2f0ca71c7642a697e5d9237"
-  packages = ["."]
-  revision = "7f0af6269940baa2c938fabe73e0d7ba41205683"
-
-[[projects]]
-  branch = "master"
   name = "github.com/syndtr/goleveldb"
   packages = [
     "leveldb",
@@ -220,21 +275,27 @@
     "leveldb/table",
     "leveldb/util"
   ]
+  pruneopts = ""
   revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
 
 [[projects]]
+  digest = "1:b9bca27d5fbe4ad1a1802706629580ce923bb14255c51375480dff1bebcbb8b2"
   name = "github.com/ugorji/go"
   packages = ["codec"]
+  pruneopts = ""
   revision = "71c2886f5a673a35f909803f38ece5810165097b"
 
 [[projects]]
   branch = "master"
+  digest = "1:2d8078b329a80bf4b71537dcd30853ff89fcafb611c94b2dfdb3d43ef1f23a1a"
   name = "github.com/wsxiaoys/terminal"
   packages = ["color"]
+  pruneopts = ""
   revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
 
 [[projects]]
   branch = "master"
+  digest = "1:51aff2b69272cb95713b74675c0817e0a0caadc20337a8d9115bc68bf1105280"
   name = "golang.org/x/crypto"
   packages = [
     "cast5",
@@ -247,44 +308,97 @@
     "openpgp/s2k",
     "ssh/terminal"
   ]
+  pruneopts = ""
   revision = "b2aa35443fbc700ab74c586ae79b81c171851023"
 
 [[projects]]
   branch = "master"
+  digest = "1:267aa6124260d0e48d1ebf8bb84dce97caa133406b3caabc076cfc7a9e85ffe4"
   name = "golang.org/x/sys"
   packages = [
     "unix",
     "windows"
   ]
+  pruneopts = ""
   revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e"
 
 [[projects]]
   branch = "v1"
+  digest = "1:e75566abfb876e81f00290ec153ff994c33bf8886134c1a38a9a9df5c15a2045"
   name = "gopkg.in/check.v1"
   packages = ["."]
+  pruneopts = ""
   revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
 
 [[projects]]
+  digest = "1:dd549e360e5a8f982a28c2bcbe667307ceffe538ed9afc7c965524f1ac285b3f"
   name = "gopkg.in/go-playground/validator.v8"
   packages = ["."]
+  pruneopts = ""
   revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
   version = "v8.18.2"
 
 [[projects]]
+  digest = "1:0e886d5a5845f58255fb57bcb4a6aae7e1f9dd7c53defe7b87d57dd4cd127545"
   name = "gopkg.in/h2non/filetype.v1"
   packages = ["types"]
+  pruneopts = ""
   revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
   version = "v1.0.1"
 
 [[projects]]
   branch = "v2"
+  digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
+  pruneopts = ""
   revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "5ab2b384766e62be84d3941971a1d8e99c637f80a2cb1482b3d9704c668b549f"
+  input-imports = [
+    "github.com/AlekSi/pointer",
+    "github.com/DisposaBoy/JsonConfigReader",
+    "github.com/awalterschulze/gographviz",
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/awserr",
+    "github.com/aws/aws-sdk-go/aws/corehandlers",
+    "github.com/aws/aws-sdk-go/aws/credentials",
+    "github.com/aws/aws-sdk-go/aws/request",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/s3",
+    "github.com/cheggaaa/pb",
+    "github.com/gin-gonic/gin",
+    "github.com/h2non/filetype/matchers",
+    "github.com/kjk/lzma",
+    "github.com/mattn/go-shellwords",
+    "github.com/mkrautz/goar",
+    "github.com/mxk/go-flowrate/flowrate",
+    "github.com/ncw/swift",
+    "github.com/ncw/swift/swifttest",
+    "github.com/pborman/uuid",
+    "github.com/pkg/errors",
+    "github.com/smira/commander",
+    "github.com/smira/flag",
+    "github.com/smira/go-aws-auth",
+    "github.com/smira/go-ftp-protocol/protocol",
+    "github.com/smira/go-xz",
+    "github.com/syndtr/goleveldb/leveldb",
+    "github.com/syndtr/goleveldb/leveldb/filter",
+    "github.com/syndtr/goleveldb/leveldb/opt",
+    "github.com/syndtr/goleveldb/leveldb/storage",
+    "github.com/syndtr/goleveldb/leveldb/util",
+    "github.com/ugorji/go/codec",
+    "github.com/wsxiaoys/terminal/color",
+    "golang.org/x/crypto/openpgp",
+    "golang.org/x/crypto/openpgp/armor",
+    "golang.org/x/crypto/openpgp/clearsign",
+    "golang.org/x/crypto/openpgp/errors",
+    "golang.org/x/crypto/openpgp/packet",
+    "golang.org/x/crypto/ssh/terminal",
+    "golang.org/x/sys/unix",
+    "gopkg.in/check.v1"
+  ]
   solver-name = "gps-cdcl"
   solver-version = 1

Makefile

@@ -1,18 +1,20 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
+ifdef TRAVIS_TAG
+	TAG=$(TRAVIS_TAG)
+else
+	TAG="$(shell git describe --tags)"
+endif
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
 PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python
 TESTS?=
 BINPATH?=$(GOPATH)/bin
 RUN_LONG_TESTS?=yes
 
-GO_1_10_AND_HIGHER=$(shell (printf '%s\n' go1.10 $(GOVERSION) | sort -cV >/dev/null 2>&1) && echo "yes")
+all: test bench check system-test
-
-all: test check system-test
 
 prepare:
-	go get -u github.com/alecthomas/gometalinter
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.17.1
-	gometalinter --install
 
 dev:
 	go get -u github.com/golang/dep/...
@@ -20,11 +22,7 @@ dev:
 
 check: system/env
 ifeq ($(RUN_LONG_TESTS), yes)
-	if [ -x travis_wait ]; then \
+	golangci-lint run
-		travis_wait gometalinter --config=linter.json ./...; \
-	else \
-		gometalinter --config=linter.json ./...; \
-	fi
 	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
 endif
 
@@ -46,24 +44,29 @@ ifeq ($(RUN_LONG_TESTS), yes)
 endif
 
 test:
-ifeq ($(GO_1_10_AND_HIGHER), yes)
 	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
-else
+
-	go test -v `go list ./... | grep -v vendor/` -gocheck.v=true
+bench:
-endif
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc:
+goxc: dev
 	rm -rf root/
 	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
 	cp man/aptly.1 root/usr/share/man/man1
 	cp completion.d/aptly root/etc/bash_completion.d/
 	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
 	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
+
+release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
+release: goxc
+	rm -rf build/
+	mkdir -p build/
+	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
 
 man:
 	make -C man
@@ -71,4 +74,4 @@ man:
 version:
 	@echo $(VERSION)
 
-.PHONY: man version
+.PHONY: man version release goxc

README.rst

@@ -64,14 +64,14 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.8+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.10+ required)::
 
     mkdir -p $GOPATH/src/github.com/aptly-dev/aptly
     git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly
     cd $GOPATH/src/github.com/aptly-dev/aptly
     make install
 
-Binary would be installed to ```$GOPATH/bin/aptly``.
+Binary would be installed to ``$GOPATH/bin/aptly``.
 
 Contributing
 ------------
@@ -90,7 +90,7 @@ Vagrant:
 Docker:
 
 -    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
--    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
 
 With configuration management systems:
 

api/publish.go

@@ -300,7 +300,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			}
 
 			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
-			if err != nil {
+			if err2 != nil {
 				c.AbortWithError(404, err2)
 				return
 			}

api/repos.go

@@ -8,6 +8,7 @@ import (
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
 )
@@ -367,3 +368,77 @@ func apiReposPackageFromDir(c *gin.Context) {
 		"FailedFiles": failedFiles,
 	})
 }
+
+// POST /repos/:name/include/:dir/:file
+func apiReposIncludePackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposIncludePackageFromDir(c)
+}
+
+// POST /repos/:name/include/:dir
+func apiReposIncludePackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
+	acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
+	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
+
+	repoTemplateString := c.Params.ByName("name")
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	var (
+		err                       error
+		verifier                  = context.GetVerifier()
+		sources, changesFiles     []string
+		failedFiles, failedFiles2 []string
+		reporter                  = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.Lock()
+	defer localRepoCollection.Unlock()
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+		repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
+		return
+	}
+
+	if !noRemoveFiles {
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -71,6 +71,9 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
 		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
 
+		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
 		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 

aptly-api.service

@@ -6,7 +6,7 @@ Documentation=https://www.aptly.info/doc/api/
 
 [Service]
 Type=simple
-ExecStart=/usr/bin/aptly serve api -no-lock -listen=127.0.0.1:8081
+ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
 
 [Install]
 WantedBy=multi-user.target

aptly/interfaces.go

@@ -69,7 +69,7 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, baseName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
@@ -130,6 +130,8 @@ type Downloader interface {
 	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
 }
 
 // ChecksumStorage is stores checksums in some (persistent) storage

cmd/api_serve.go

@@ -58,7 +58,25 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-		os.Remove(file)
+
+		var stat os.FileInfo
+		stat, err = os.Stat(file)
+		shouldRemove := true
+
+		if err == nil && stat.Mode()&os.ModeSocket == os.ModeSocket {
+			shouldRemove = false
+		}
+
+		if err != nil && os.IsNotExist(err) {
+			shouldRemove = false
+		}
+
+		if shouldRemove {
+			err = os.Remove(file)
+			if err != nil {
+				fmt.Printf("Warning: error removing file %s: %s\n", file, err)
+			}
+		}
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)

cmd/cmd.go

@@ -119,7 +119,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
-	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\" for external gpg or \"internal\" for Go internal implementation)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/db_cleanup.go

@@ -59,6 +59,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
 	}
@@ -90,6 +92,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
 	}
@@ -118,6 +122,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
 	}
@@ -150,6 +156,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
@@ -192,6 +200,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	context.CollectionFactory().Flush()
+
 	// now, build a list of files that should be present in Repository (package pool)
 	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())

cmd/mirror_create.go

@@ -19,6 +19,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -36,7 +37,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs)
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -94,6 +95,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")

cmd/mirror_edit.go

@@ -33,6 +33,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.Filter = flag.Value.String()
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
@@ -101,6 +103,7 @@ Example:
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

cmd/mirror_update.go

@@ -54,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}

cmd/package_show.go

@@ -79,7 +79,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 	result := q.Query(context.CollectionFactory().PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false)
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
 		w.Flush()
 		fmt.Printf("\n")
 

cmd/repo_include.go

@@ -1,16 +1,11 @@
 package cmd
 
 import (
-	"bytes"
 	"fmt"
-	"os"
-	"path/filepath"
-	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/query"
-	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -35,11 +30,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
 	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
-
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
-	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
-	if err != nil {
-		return fmt.Errorf("error parsing -repo template: %s", err)
-	}
 
 	uploaders := (*deb.Uploaders)(nil)
 	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
@@ -59,143 +50,15 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
 
-	var changesFiles, failedFiles, processedFiles []string
+	var changesFiles, failedFiles, failedFiles2 []string
 
 	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
-
+	_, failedFiles2, err = deb.ImportChangesFiles(
-	for _, path := range changesFiles {
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplateString,
-		var changes *deb.Changes
+		context.Progress(), context.CollectionFactory().LocalRepoCollection(), context.CollectionFactory().PackageCollection(),
-
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(),
-		changes, err = deb.NewChanges(path)
+		uploaders, query.Parse)
-		if err != nil {
+	failedFiles = append(failedFiles, failedFiles2...)
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", path, err)
-			continue
-		}
-
-		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		err = changes.Prepare()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		repoName := &bytes.Buffer{}
-		err = repoTemplate.Execute(repoName, changes.Stanza)
-		if err != nil {
-			return fmt.Errorf("error applying template to repo: %s", err)
-		}
-
-		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
-
-		var repo *deb.LocalRepo
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		currentUploaders := uploaders
-		if repo.Uploaders != nil {
-			currentUploaders = repo.Uploaders
-			for i := range currentUploaders.Rules {
-				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
-				if err != nil {
-					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
-				}
-			}
-		}
-
-		if currentUploaders != nil {
-			if err = currentUploaders.IsAllowed(changes); err != nil {
-				failedFiles = append(failedFiles, path)
-				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
-					changes.ChangesName, changes.SignatureKeys, err)
-				changes.Cleanup()
-				continue
-			}
-		}
-
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return fmt.Errorf("unable to load repo: %s", err)
-		}
-
-		var list *deb.PackageList
-		list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-		if err != nil {
-			return fmt.Errorf("unable to load packages: %s", err)
-		}
-
-		packageFiles, otherFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
-
-		var restriction deb.PackageQuery
-
-		restriction, err = changes.PackageQuery()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		var processedFiles2, failedFiles2 []string
-
-		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())
-
-		if err != nil {
-			return fmt.Errorf("unable to import package files: %s", err)
-		}
-
-		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-		if err != nil {
-			return fmt.Errorf("unable to save: %s", err)
-		}
-
-		err = changes.Cleanup()
-		if err != nil {
-			return err
-		}
-
-		for _, file := range failedFiles2 {
-			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		for _, file := range processedFiles2 {
-			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		for _, file := range otherFiles {
-			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		processedFiles = append(processedFiles, path)
-	}
-
-	if !noRemoveFiles {
-		processedFiles = utils.StrSliceDeduplicate(processedFiles)
-
-		for _, file := range processedFiles {
-			err = os.Remove(file)
-			if err != nil {
-				return fmt.Errorf("unable to remove file: %s", err)
-			}
-		}
-	}
 
 	if len(failedFiles) > 0 {
 		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")

completion.d/aptly

@@ -156,7 +156,7 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
@@ -164,7 +164,7 @@ _aptly()
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi

context/context.go

@@ -387,23 +387,42 @@ func (context *AptlyContext) pgpProvider() string {
 		provider = context.config().GpgProvider
 	}
 
-	if !(provider == "gpg" || provider == "internal") { // nolint: goconst
+	switch provider {
+	case "gpg": // nolint: goconst
+	case "gpg1": // nolint: goconst
+	case "gpg2": // nolint: goconst
+	case "internal": // nolint: goconst
+	default:
 		Fatal(fmt.Errorf("unknown gpg provider: %v", provider))
 	}
 
 	return provider
 }
 
+func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
+	switch context.pgpProvider() {
+	case "gpg1":
+		return pgp.GPG1Finder()
+	case "gpg2":
+		return pgp.GPG2Finder()
+	case "gpg":
+		return pgp.GPGDefaultFinder()
+	}
+
+	panic("uknown GPG provider type")
+}
+
 // GetSigner returns Signer with respect to provider
 func (context *AptlyContext) GetSigner() pgp.Signer {
 	context.Lock()
 	defer context.Unlock()
 
-	if context.pgpProvider() == "gpg" { // nolint: goconst
+	provider := context.pgpProvider()
-		return pgp.NewGpgSigner()
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoSigner{}
 	}
 
-	return &pgp.GoSigner{}
+	return pgp.NewGpgSigner(context.getGPGFinder(provider))
 }
 
 // GetVerifier returns Verifier with respect to provider
@@ -411,11 +430,12 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
 	context.Lock()
 	defer context.Unlock()
 
-	if context.pgpProvider() == "gpg" { // nolint: goconst
+	provider := context.pgpProvider()
-		return pgp.NewGpgVerifier()
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoVerifier{}
 	}
 
-	return &pgp.GoVerifier{}
+	return pgp.NewGpgVerifier(context.getGPGFinder(provider))
 }
 
 // UpdateFlags sets internal copy of flags in the context

deb/changes.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -8,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
@@ -93,8 +95,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		text = input
 	}
 
-	reader := NewControlFileReader(text)
+	reader := NewControlFileReader(text, false, false)
-	c.Stanza, err = reader.ReadStanza(false)
+	c.Stanza, err = reader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -164,7 +166,7 @@ func (c *Changes) Cleanup() error {
 }
 
 // PackageQuery returns query that every package should match to be included
-func (c *Changes) PackageQuery() (PackageQuery, error) {
+func (c *Changes) PackageQuery() PackageQuery {
 	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
 	for _, arch := range c.Architectures {
 		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
@@ -215,7 +217,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
 	}
 
-	return &AndQuery{L: archQuery, R: nameQuery}, nil
+	return &AndQuery{L: archQuery, R: nameQuery}
 }
 
 // GetField implements PackageLike interface
@@ -288,3 +290,143 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 
 	return
 }
+
+// ImportChangesFiles imports referenced files in changes files into local repository
+func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
+	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
+	}
+	for _, path := range changesFiles {
+		var changes *Changes
+
+		changes, err = NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		if progress != nil {
+			progress.Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+		}
+
+		var repo *LocalRepo
+		repo, err = localRepoCollection.ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = parseQuery(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return nil, nil, fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		var list *PackageList
+		list, err = NewPackageListFromRefList(repo.RefList(), packageCollection, progress)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, otherFiles, _ := CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		restriction := changes.PackageQuery()
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
+			packageCollection, reporter, restriction, checksumStorage)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
+
+		err = localRepoCollection.Update(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err = os.Remove(file)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	return processedFiles, failedFiles, nil
+}

deb/changes_test.go

@@ -4,24 +4,52 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type ChangesSuite struct {
-	Dir, Path string
+	Dir, Path           string
+	Reporter            aptly.ResultReporter
+	db                  database.Storage
+	localRepoCollection *LocalRepoCollection
+	packageCollection   *PackageCollection
+	packagePool         aptly.PackagePool
+	checksumStorage     aptly.ChecksumStorage
+	progress            aptly.Progress
 }
 
 var _ = Suite(&ChangesSuite{})
 
 func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Reporter = &aptly.RecordingResultReporter{
+		Warnings:     []string{},
+		AddedLines:   []string{},
+		RemovedLines: []string{},
+	}
 	s.Dir = c.MkDir()
 	s.Path = filepath.Join(s.Dir, "calamares.changes")
-
+	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
-	f, err := os.Create(s.Path)
 	c.Assert(err, IsNil)
 
-	f.WriteString(changesFile)
+	s.db, _ = database.NewOpenDB(c.MkDir())
-	f.Close()
+	s.localRepoCollection = NewLocalRepoCollection(s.db)
+	s.packageCollection = NewPackageCollection(s.db)
+
+	s.checksumStorage = files.NewMockChecksumStorage()
+	s.packagePool = files.NewPackagePool(s.Dir, false)
+	s.progress = console.NewProgress()
+	s.progress.Start()
+}
+
+func (s *ChangesSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
+	s.db.Close()
 }
 
 func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -44,6 +72,73 @@ func (s *ChangesSuite) TestParseAndVerify(c *C) {
 	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
 }
 
+func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
+	changesFiles, failedFiles := CollectChangesFiles([]string{"testdata/changes"}, s.Reporter)
+
+	c.Check(failedFiles, HasLen, 0)
+	c.Check(changesFiles, DeepEquals, []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	})
+}
+
+func (s *ChangesSuite) TestImportChangesFiles(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	origFailedFiles := []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.0_i386.deb",
+	}
+	origProcessedFiles := []string{
+		"testdata/changes/hardlink_0.2.1.dsc",
+		"testdata/changes/hardlink_0.2.1.tar.gz",
+		"testdata/changes/hardlink_0.2.1_amd64.deb",
+		"testdata/changes/hardlink_0.2.1_amd64.buildinfo",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	}
+
+	var expectedProcessedFiles, expectedFailedFiles []string
+
+	for _, path := range origFailedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedFailedFiles = append(expectedFailedFiles, filename)
+	}
+
+	for _, path := range origProcessedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedProcessedFiles = append(expectedProcessedFiles, filename)
+	}
+
+	changesFiles, failedFiles := CollectChangesFiles([]string{s.Dir}, s.Reporter)
+	c.Check(failedFiles, HasLen, 0)
+
+	processedFiles, failedFiles, err := ImportChangesFiles(
+		append(changesFiles, "testdata/changes/notexistent.changes"),
+		s.Reporter, true, true, false, false, &NullVerifier{},
+		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
+	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
+}
+
+func (s *ChangesSuite) TestPrepare(c *C) {
+	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
+	c.Assert(err, IsNil)
+	err = changes.Prepare()
+	c.Assert(err, IsNil)
+
+	_, err = os.Stat(filepath.Join(changes.TempDir, "hardlink_0.2.1_amd64.changes"))
+	c.Check(err, IsNil)
+}
+
 func (s *ChangesSuite) TestPackageQuery(c *C) {
 	changes, err := NewChanges(s.Path)
 	c.Assert(err, IsNil)
@@ -51,42 +146,7 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 	err = changes.VerifyAndParse(true, true, &NullVerifier{})
 	c.Check(err, IsNil)
 
-	q, err := changes.PackageQuery()
+	q := changes.PackageQuery()
-	c.Check(err, IsNil)
-
 	c.Check(q.String(), Equals,
 		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
 }
-
-var changesFile = `Format: 1.8
-Date: Thu, 27 Nov 2014 13:24:53 +0000
-Source: calamares
-Binary: calamares calamares-dbg
-Architecture: source amd64
-Version: 0+git20141127.99
-Distribution: sid
-Urgency: medium
-Maintainer: Rohan Garg <rohan@kde.org>
-Changed-By: Rohan <rohan@kde.org>
-Description:
- calamares  - distribution-independent installer framework
- calamares-dbg - distribution-independent installer framework -- debug symbols
-Changes:
- calamares (0+git20141127.99) sid; urgency=medium
- .
-   * Update from git
-Checksums-Sha1:
- 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
- 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
- d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
- a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Checksums-Sha256:
- 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
- 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
- 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
- 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Files:
- 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
- 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
- 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
- e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/contents.go

@@ -7,7 +7,7 @@ import (
 	"io"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 )
 
 // ContentsIndex calculates mapping from files to packages, with sorting and aggregation

deb/deb.go

@@ -11,12 +11,12 @@ import (
 	"strings"
 
 	"github.com/h2non/filetype/matchers"
-	"github.com/mkrautz/goar"
+	ar "github.com/mkrautz/goar"
 	"github.com/pkg/errors"
 
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/kjk/lzma"
 	"github.com/smira/go-xz"
-	"github.com/smira/lzma"
 )
 
 // Source kinds
@@ -26,6 +26,8 @@ const (
 	SourceRemoteRepo = "repo"
 )
 
+type parseQuery func(string) (PackageQuery, error)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -87,8 +89,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
-					reader := NewControlFileReader(untar)
+					reader := NewControlFileReader(untar, false, false)
-					stanza, err := reader.ReadStanza(false)
+					stanza, err := reader.ReadStanza()
 					if err != nil {
 						return nil, err
 					}
@@ -127,8 +129,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		text = file
 	}
 
-	reader := NewControlFileReader(text)
+	reader := NewControlFileReader(text, false, false)
-	stanza, err := reader.ReadStanza(false)
+	stanza, err := reader.ReadStanza()
 	if err != nil {
 		return nil, err
 	}

deb/format.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"io"
+	"sort"
 	"strings"
 	"unicode"
 )
@@ -88,6 +89,9 @@ var (
 		"Directory",
 		"Files",
 	}
+	canonicalOrderInstaller = []string{
+		"",
+	}
 )
 
 // Copy returns copy of Stanza
@@ -101,6 +105,9 @@ func (s Stanza) Copy() (result Stanza) {
 
 func isMultilineField(field string, isRelease bool) bool {
 	switch field {
+	// file without a section
+	case "":
+		return true
 	case "Description":
 		return true
 	case "Files":
@@ -127,26 +134,33 @@ func isMultilineField(field string, isRelease bool) bool {
 	return false
 }
 
-// Write single field from Stanza to writer
+// Write single field from Stanza to writer.
+//
+//nolint: interfacer
 func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
 	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
-		if !strings.HasSuffix(value, "\n") {
+		if field != "" && !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
 
-		if field != "Description" {
+		if field != "Description" && field != "" {
 			value = "\n" + value
 		}
-		_, err = w.WriteString(field + ":" + value)
+
+		if field != "" {
+			_, err = w.WriteString(field + ":" + value)
+		} else {
+			_, err = w.WriteString(value)
+		}
 	}
 
 	return
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
 	canonicalOrder := canonicalOrderBinary
 	if isSource {
 		canonicalOrder = canonicalOrderSource
@@ -154,6 +168,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	if isRelease {
 		canonicalOrder = canonicalOrderRelease
 	}
+	if isInstaller {
+		canonicalOrder = canonicalOrderInstaller
+	}
 
 	for _, field := range canonicalOrder {
 		value, ok := s[field]
@@ -166,10 +183,21 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		}
 	}
 
-	for field, value := range s {
+	// no extra fields in installer
-		err := writeField(w, field, value, isRelease)
+	if !isInstaller {
-		if err != nil {
+		// Print extra fields in deterministic order (alphabetical)
-			return err
+		keys := make([]string, len(s))
+		i := 0
+		for field := range s {
+			keys[i] = field
+			i++
+		}
+		sort.Strings(keys)
+		for _, field := range keys {
+			err := writeField(w, field, s[field], isRelease)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -212,22 +240,28 @@ func canonicalCase(field string) string {
 
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
-	scanner *bufio.Scanner
+	scanner     *bufio.Scanner
+	isRelease   bool
+	isInstaller bool
 }
 
 // NewControlFileReader creates ControlFileReader, it wraps with buffering
-func NewControlFileReader(r io.Reader) *ControlFileReader {
+func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
 	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
 	scnr.Buffer(nil, MaxFieldSize)
 
-	return &ControlFileReader{scanner: scnr}
+	return &ControlFileReader{
+		scanner:     scnr,
+		isRelease:   isRelease,
+		isInstaller: isInstaller,
+	}
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
+func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
-	lastFieldMultiline := false
+	lastFieldMultiline := c.isInstaller
 
 	for c.scanner.Scan() {
 		line := c.scanner.Text()
@@ -240,7 +274,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 			continue
 		}
 
-		if line[0] == ' ' || line[0] == '\t' {
+		if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
@@ -252,7 +286,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			lastFieldMultiline = isMultilineField(lastField, isRelease)
+			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -15,6 +15,10 @@ type ControlFileSuite struct {
 
 var _ = Suite(&ControlFileSuite{})
 
+const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list
+9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+`
+
 const controlFile = `Package: bti
 Binary: bti
 Version: 032-1
@@ -83,15 +87,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
 
-	stanza1, err := r.ReadStanza(false)
+	stanza1, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza(false)
+	stanza3, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
@@ -103,27 +107,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
-	stanza, err := r.ReadStanza(false)
+	stanza, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, true, false)
+	err = stanza.Copy().WriteTo(w, true, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
 
 	str := buf.String()
 
-	r = NewControlFileReader(buf)
+	r = NewControlFileReader(buf, false, false)
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
+	s.reader = bytes.NewBufferString(installerFile)
+	r := NewControlFileReader(s.reader, false, true)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, false, false, true)
+	c.Assert(err, IsNil)
+	err = w.Flush()
+	c.Assert(err, IsNil)
+
+	r = NewControlFileReader(buf, false, true)
+	stanza2, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Assert(stanza2, DeepEquals, stanza)
+}
+
 func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("Package"), Equals, "Package")
 	c.Check(canonicalCase("package"), Equals, "Package")
@@ -141,8 +165,8 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 	c.Assert(err, IsNil)
 	defer f.Close()
 
-	r := NewControlFileReader(f)
+	r := NewControlFileReader(f, false, false)
-	stanza, e := r.ReadStanza(false)
+	stanza, e := r.ReadStanza()
 	c.Assert(e, IsNil)
 	c.Assert(len(stanza["Provides"]), Equals, 586929)
 }
@@ -150,9 +174,9 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
-		r := NewControlFileReader(reader)
+		r := NewControlFileReader(reader, false, false)
 		for {
-			s, e := r.ReadStanza(false)
+			s, e := r.ReadStanza()
 			if s == nil && e == nil {
 				break
 			}

deb/index_files.go

@@ -29,7 +29,8 @@ type indexFile struct {
 	discardable   bool
 	compressable  bool
 	onlyGzip      bool
-	signable      bool
+	clearSign     bool
+	detachedSign  bool
 	acquireByHash bool
 	relativePath  string
 	tempFilename  string
@@ -77,14 +78,17 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	file.tempFile.Close()
 
 	exts := []string{""}
+	cksumExts := exts
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		cksumExts = exts
 		if file.onlyGzip {
 			exts = []string{".gz"}
+			cksumExts = []string{"", ".gz"}
 		}
 	}
 
-	for _, ext := range exts {
+	for _, ext := range cksumExts {
 		var checksumInfo utils.ChecksumInfo
 
 		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
@@ -133,34 +137,42 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		}
 	}
 
-	if file.signable && signer != nil {
+	if signer != nil {
-		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if file.detachedSign {
-		if err != nil {
+			err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
-			return fmt.Errorf("unable to detached sign file: %s", err)
+			if err != nil {
+				return fmt.Errorf("unable to detached sign file: %s", err)
+			}
+
+			if file.parent.suffix != "" {
+				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+					filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			}
+
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+				file.tempFilename+".gpg")
+			if err != nil {
+				return fmt.Errorf("unable to publish file: %s", err)
+			}
+
 		}
 
-		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if file.clearSign {
-		if err != nil {
+			err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-			return fmt.Errorf("unable to clearsign file: %s", err)
+			if err != nil {
-		}
+				return fmt.Errorf("unable to clearsign file: %s", err)
+			}
 
-		if file.parent.suffix != "" {
+			if file.parent.suffix != "" {
-			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
-				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+					filepath.Join(file.parent.basePath, "In"+file.relativePath)
-			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+			}
-				filepath.Join(file.parent.basePath, "In"+file.relativePath)
-		}
 
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
-			file.tempFilename+".gpg")
+				filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
+			if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
+				return fmt.Errorf("unable to publish file: %s", err)
-		}
+			}
-
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
-			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
 		}
 	}
 
@@ -230,11 +242,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 	}
 }
 
-func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false
 	}
-	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
+	key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -244,6 +256,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		} else {
 			if udeb {
 				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else if installer {
+				relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
 			} else {
 				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
 			}
@@ -252,8 +266,9 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		file = &indexFile{
 			parent:        files,
 			discardable:   false,
-			compressable:  true,
+			compressable:  !installer,
-			signable:      false,
+			detachedSign:  installer,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -287,7 +302,8 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 			parent:        files,
 			discardable:   udeb,
 			compressable:  false,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -318,7 +334,8 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 			discardable:   true,
 			compressable:  true,
 			onlyGzip:      true,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -349,7 +366,8 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile
 			discardable:   true,
 			compressable:  true,
 			onlyGzip:      true,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -365,19 +383,20 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 		parent:       files,
 		discardable:  false,
 		compressable: false,
-		signable:     true,
+		detachedSign: true,
+		clearSign:    true,
 		relativePath: "Release",
 	}
 }
 
-func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
 		progress.InitBar(int64(len(files.indexes)), false)
 		defer progress.ShutdownBar()
 	}
 
 	for _, file := range files.indexes {
-		err = file.Finalize(nil)
+		err = file.Finalize(signer)
 		if err != nil {
 			return
 		}

deb/local.go

@@ -2,12 +2,13 @@ package deb
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"log"
 	"sync"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -93,46 +94,68 @@ func (repo *LocalRepo) RefKey() []byte {
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*LocalRepo
+	cache map[string]*LocalRepo
 }
 
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
-	result := &LocalRepoCollection{
+	return &LocalRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*LocalRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("L"))
+func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, unique bool) []*LocalRepo {
-	result.list = make([]*LocalRepo, 0, len(blobs))
+	result := []*LocalRepo(nil)
-
+	for _, r := range collection.cache {
-	for _, blob := range blobs {
+		if filter(r) {
-		r := &LocalRepo{}
+			result = append(result, r)
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding repo: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+		r := &LocalRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding local repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
-	for _, r := range collection.list {
+	_, err := collection.ByName(repo.Name)
-		if r.Name == repo.Name {
+
-			return fmt.Errorf("local repo with name %s already exists", repo.Name)
+	if err == nil {
-		}
+		return fmt.Errorf("local repo with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -167,58 +190,66 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 
 // ByName looks up repository by name
 func (collection *LocalRepoCollection) ByName(name string) (*LocalRepo, error) {
-	for _, r := range collection.list {
+	result := collection.search(func(r *LocalRepo) bool { return r.Name == name }, true)
-		if r.Name == name {
+	if len(result) == 0 {
-			return r, nil
+		return nil, fmt.Errorf("local repo with name %s not found", name)
-		}
 	}
-	return nil, fmt.Errorf("local repo with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
-	for _, r := range collection.list {
+	if r, ok := collection.cache[uuid]; ok {
-		if r.UUID == uuid {
+		return r, nil
-			return r, nil
-		}
 	}
-	return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+
+	key := (&LocalRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	r := &LocalRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &LocalRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding repo: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *LocalRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("L")))
 }
 
 // Drop removes remote repo from collection
 func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
-	repoPosition := -1
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
 		panic("local repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+	delete(collection.cache, repo.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/local_test.go

@@ -124,6 +124,11 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewLocalRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 

deb/package.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	gocontext "context"
 	"encoding/json"
 	"fmt"
 	"path/filepath"
@@ -26,6 +27,8 @@ type Package struct {
 	Provides []string
 	// Hash of files section
 	FilesHash uint64
+	// Is this package a dummy installer package
+	IsInstaller bool
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
@@ -43,9 +46,10 @@ type Package struct {
 
 // Package types
 const (
-	PackageTypeBinary = "deb"
+	PackageTypeBinary    = "deb"
-	PackageTypeUdeb   = "udeb"
+	PackageTypeUdeb      = "udeb"
-	PackageTypeSource = "source"
+	PackageTypeSource    = "source"
+	PackageTypeInstaller = "installer"
 )
 
 // Special arhictectures
@@ -168,6 +172,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package {
 	return p
 }
 
+// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file
+func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) {
+	p := &Package{
+		Name:         "installer",
+		Architecture: architecture,
+		IsInstaller:  true,
+		V06Plus:      true,
+		extra:        &Stanza{},
+		deps:         &PackageDependencies{},
+	}
+
+	files := make(PackageFiles, 0)
+	files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	for i := range files {
+		files[i].downloadPath = relPath
+
+		url := repo.PackageURL(files[i].DownloadURL()).String()
+		var size int64
+		size, err = d.GetLength(gocontext.TODO(), url)
+		if err != nil {
+			return nil, err
+		}
+
+		files[i].Checksums.Size = size
+	}
+
+	p.UpdateFiles(files)
+	return p, nil
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -509,6 +548,12 @@ func (p *Package) Stanza() (result Stanza) {
 		if len(sha512) > 0 {
 			result["Checksums-Sha512"] = strings.Join(sha512, "")
 		}
+	} else if p.IsInstaller {
+		sha256 := []string{}
+		for _, f := range p.Files() {
+			sha256 = append(sha256, fmt.Sprintf("%s  %s", f.Checksums.SHA256, f.Filename))
+		}
+		result[""] = strings.Join(sha256, "\n")
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -563,11 +608,7 @@ func (p *Package) Equals(p2 *Package) bool {
 
 // LinkFromPool links package file from pool to dist's pool location
 func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
-	prefix, component string, force bool) error {
+	prefix, relPath string, force bool) error {
-	poolDir, err := p.PoolDirectory()
-	if err != nil {
-		return err
-	}
 
 	for i, f := range p.Files() {
 		sourcePoolPath, err := f.GetPoolPath(packagePool)
@@ -575,7 +616,6 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
 		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)

deb/package_files.go

@@ -88,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
 
-func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+// ParseSumField populates PackageFiles by parsing given input
+func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) {
 	for _, line := range strings.Split(input, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
@@ -96,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 		}
 		parts := strings.Fields(line)
 
-		if len(parts) < 3 {
+		if withSize && len(parts) < 3 || !withSize && len(parts) < 2 {
 			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
 		}
 
-		size, err := strconv.ParseInt(parts[1], 10, 64)
+		var size int64
-		if err != nil {
+		var err error
-			return nil, fmt.Errorf("unable to parse size: %s", err)
+		if withSize {
+			size, err = strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse size: %s", err)
+			}
 		}
 
-		filename := filepath.Base(parts[len(parts)-1])
+		filename := parts[len(parts)-1]
+		if onlyBasePath {
+			filename = filepath.Base(filename)
+		}
 
 		found := false
 		pos := 0
@@ -133,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
 	var err error
 
-	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}

deb/package_test.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/http"
 
 	. "gopkg.in/check.v1"
 )
@@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
+	s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza()
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
 }
 
+func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
+	downloader.ExpectResponse("", "udeb.list")
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza()
+	p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader)
+	c.Check(err, IsNil)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
+	c.Check(p.IsInstaller, Equals, true)
+	c.Check(p.Name, Equals, "installer")
+	c.Check(p.Version, Equals, "")
+	c.Check(p.Architecture, Equals, "i386")
+	c.Check(p.Files(), HasLen, 2)
+	c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs")
+	c.Check(p.Files()[0].Checksums.Size, Equals, int64(14))
+	c.Check(p.Files()[1].Filename, Equals, "./udeb.list")
+	c.Check(p.Files()[1].Checksums.Size, Equals, int64(9))
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
@@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -490,3 +514,6 @@ Size: 29188
 MD5sum: ae70341c4d96dcded89fa670bcfea31e
 SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
 SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`
+
+const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list`

deb/publish.go

@@ -14,7 +14,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -569,7 +569,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
-			indexes.PackageIndex(component, arch, false)
+			indexes.PackageIndex(component, arch, false, false)
 		}
 
 		if progress != nil {
@@ -585,19 +585,26 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				progress.AddBar(1)
 			}
 
-			matches := false
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					matches = true
+					hadUdebs = hadUdebs || pkg.IsUdeb
-					break
-				}
-			}
 
-			if matches {
+					var relPath string
-				hadUdebs = hadUdebs || pkg.IsUdeb
+					if !pkg.IsInstaller {
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
+						poolDir, err2 := pkg.PoolDirectory()
-				if err != nil {
+						if err2 != nil {
-					return err
+							return err2
+						}
+						relPath = filepath.Join("pool", component, poolDir)
+					} else {
+						relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
+					}
+
+					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite)
+					if err != nil {
+						return err
+					}
+					break
 				}
 			}
 
@@ -612,7 +619,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
-					if !p.SkipContents {
+					if !p.SkipContents && !pkg.IsInstaller {
 						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
 						qualifiedName := []byte(pkg.QualifiedName())
 						contents := pkg.Contents(packagePool, progress)
@@ -629,12 +636,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						}
 					}
 
-					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller)
 					if err != nil {
 						return err
 					}
@@ -687,7 +694,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			// For all architectures, pregenerate .udeb indexes
 			for _, arch := range p.Architectures {
-				indexes.PackageIndex(component, arch, true)
+				indexes.PackageIndex(component, arch, true, false)
 			}
 		}
 
@@ -710,7 +717,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
 				}
 
-				err = release.WriteTo(bufWriter, false, true)
+				err = release.WriteTo(bufWriter, false, true, false)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -742,7 +749,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		progress.Printf("Finalizing metadata files...\n")
 	}
 
-	err = indexes.FinalizeAll(progress)
+	err = indexes.FinalizeAll(progress, signer)
 	if err != nil {
 		return err
 	}
@@ -791,7 +798,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter, false, true)
+	err = release.WriteTo(bufWriter, false, true, false)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -852,28 +859,34 @@ type PublishedRepoCollection struct {
 
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
-	result := &PublishedRepoCollection{
+	return &PublishedRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("U"))
+func (collection *PublishedRepoCollection) loadList() {
-	result.list = make([]*PublishedRepo, 0, len(blobs))
+	if collection.list != nil {
+		return
+	}
+
+	blobs := collection.db.FetchByPrefix([]byte("U"))
+	collection.list = make([]*PublishedRepo, 0, len(blobs))
 
 	for _, blob := range blobs {
 		r := &PublishedRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding published repo: %s\n", err)
 		} else {
-			result.list = append(result.list, r)
+			collection.list = append(collection.list, r)
 		}
 	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
+	collection.loadList()
+
 	if collection.CheckDuplicate(repo) != nil {
 		return fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s already exists", repo.Storage, repo.Prefix, repo.Distribution)
 	}
@@ -889,6 +902,8 @@ func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 
 // CheckDuplicate verifies that there's no published repo with the same name
 func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution && r.Storage == repo.Storage {
 			return r
@@ -978,6 +993,8 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 // ByStoragePrefixDistribution looks up repository by storage, prefix & distribution
 func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage, prefix, distribution string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == prefix && r.Distribution == distribution && r.Storage == storage {
 			return r, nil
@@ -991,6 +1008,8 @@ func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage,
 
 // ByUUID looks up repository by uuid
 func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.UUID == uuid {
 			return r, nil
@@ -1001,6 +1020,8 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 
 // BySnapshot looks up repository by snapshot source
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceSnapshot {
@@ -1021,6 +1042,8 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 
 // ByLocalRepo looks up repository by local repo source
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceLocalRepo {
@@ -1041,18 +1064,21 @@ func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*Publi
 
 // ForEach runs method for each repository
 func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("U"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &PublishedRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding published repo: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *PublishedRepoCollection) Len() int {
+	collection.loadList()
+
 	return len(collection.list)
 }
 
@@ -1060,6 +1086,8 @@ func (collection *PublishedRepoCollection) Len() int {
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
 	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
 
+	collection.loadList()
+
 	var err error
 	referencedFiles := map[string][]string{}
 
@@ -1141,6 +1169,9 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
 	force, skipCleanup bool) error {
+
+	collection.loadList()
+
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1171,7 +1202,10 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 
 	err = repo.RemoveFiles(publishedStorageProvider, removePrefix, removePoolComponents, progress)
 	if err != nil {
-		return err
+		if !force {
+			return fmt.Errorf("published files removal failed, use -force-drop to override: %s", err)
+		}
+		// ignore error with -force-drop
 	}
 
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =

deb/publish_test.go

@@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 
-	cfr := NewControlFileReader(rf)
+	cfr := NewControlFileReader(rf, true, false)
-	st, err := cfr.ReadStanza(true)
+	st, err := cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(pf)
+	cfr = NewControlFileReader(pf, false, false)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza(false)
+		st, err = cfr.ReadStanza()
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza(false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
 	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(drf)
+	cfr = NewControlFileReader(drf, true, false)
-	st, err = cfr.ReadStanza(true)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")

deb/remote.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bytes"
 	gocontext "context"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -20,7 +21,7 @@ import (
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -68,6 +69,8 @@ type RemoteRepo struct {
 	DownloadSources bool
 	// Should we download .udebs?
 	DownloadUdebs bool
+	// Should we download installer files?
+	DownloadInstaller bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -78,16 +81,17 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:            uuid.New(),
+		UUID:              uuid.New(),
-		Name:            name,
+		Name:              name,
-		ArchiveRoot:     archiveRoot,
+		ArchiveRoot:       archiveRoot,
-		Distribution:    distribution,
+		Distribution:      distribution,
-		Components:      components,
+		Components:        components,
-		Architectures:   architectures,
+		Architectures:     architectures,
-		DownloadSources: downloadSources,
+		DownloadSources:   downloadSources,
-		DownloadUdebs:   downloadUdebs,
+		DownloadUdebs:     downloadUdebs,
+		DownloadInstaller: downloadInstaller,
 	}
 
 	err := result.prepare()
@@ -140,6 +144,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadUdebs {
 		srcFlag += " [udeb]"
 	}
+	if repo.DownloadInstaller {
+		srcFlag += " [installer]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -243,6 +250,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
 	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
 }
 
+// InstallerPath returns path of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -311,8 +324,8 @@ ok:
 
 	defer release.Close()
 
-	sreader := NewControlFileReader(release)
+	sreader := NewControlFileReader(release, true, false)
-	stanza, err := sreader.ReadStanza(true)
+	stanza, err := sreader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -409,7 +422,7 @@ ok:
 }
 
 // DownloadPackageIndexes downloads & parses package index files
-func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
 	ignoreMismatch bool, maxTries int) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
@@ -420,39 +433,85 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""})
 		if repo.DownloadSources {
-			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture})
 				if repo.DownloadUdebs {
-					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture})
+				}
+				if repo.DownloadInstaller {
+					packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture})
 				}
 			}
 			if repo.DownloadSources {
-				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"})
 			}
 		}
 	}
 
 	for _, info := range packagesPaths {
-		path, kind := info[0], info[1]
+		path, kind, component, architecture := info[0], info[1], info[2], info[3]
 		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+
+		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
-			return err
+			if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
+				// checking if gpg file is only needed when checksums matches are required.
+				// otherwise there actually has been no candidate found and we can continue
+				if ignoreMismatch {
+					continue
+				}
+
+				// some repos do not have installer hashsum file listed in release file but provide a separate gpg file
+				hashsumPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path}).String()
+				packagesFile, err = http.DownloadTemp(gocontext.TODO(), d, hashsumPath)
+				if err != nil {
+					if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+						// installer files are not available in all components and architectures
+						// so ignore it if not found
+						continue
+					}
+
+					return err
+				}
+
+				if verifier != nil {
+					hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
+					var filesig *os.File
+					filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
+					if err != nil {
+						return err
+					}
+
+					err = verifier.VerifyDetachedSignature(filesig, packagesFile, false)
+					if err != nil {
+						return err
+					}
+
+					_, err = packagesFile.Seek(0, 0)
+				}
+
+				packagesReader = packagesFile
+			}
+
+			if err != nil {
+				return err
+			}
 		}
 		defer packagesFile.Close()
 
 		stat, _ := packagesFile.Stat()
 		progress.InitBar(stat.Size(), true)
 
-		sreader := NewControlFileReader(packagesReader)
+		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
 		for {
-			stanza, err := sreader.ReadStanza(false)
+			stanza, err := sreader.ReadStanza()
 			if err != nil {
 				return err
 			}
@@ -474,15 +533,18 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				if err != nil {
 					return err
 				}
-			}
+			} else if kind == PackageTypeInstaller {
-			err = repo.packageList.Add(p)
+				p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d)
-			if err != nil {
+				if err != nil {
-				if _, ok := err.(*PackageConflictError); ok {
-					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
-				} else {
 					return err
 				}
 			}
+			err = repo.packageList.Add(p)
+			if _, ok := err.(*PackageConflictError); ok {
+				progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+			} else if err != nil {
+				return err
+			}
 		}
 
 		progress.ShutdownBar()
@@ -654,46 +716,68 @@ func (repo *RemoteRepo) RefKey() []byte {
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*RemoteRepo
+	cache map[string]*RemoteRepo
 }
 
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
-	result := &RemoteRepoCollection{
+	return &RemoteRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*RemoteRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("R"))
+func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, unique bool) []*RemoteRepo {
-	result.list = make([]*RemoteRepo, 0, len(blobs))
+	result := []*RemoteRepo(nil)
-
+	for _, r := range collection.cache {
-	for _, blob := range blobs {
+		if filter(r) {
-		r := &RemoteRepo{}
+			result = append(result, r)
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+		r := &RemoteRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding remote repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
-	for _, r := range collection.list {
+	_, err := collection.ByName(repo.Name)
-		if r.Name == repo.Name {
+
-			return fmt.Errorf("mirror with name %s already exists", repo.Name)
+	if err == nil {
-		}
+		return fmt.Errorf("mirror with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -728,58 +812,65 @@ func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo) error {
 
 // ByName looks up repository by name
 func (collection *RemoteRepoCollection) ByName(name string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
+	result := collection.search(func(r *RemoteRepo) bool { return r.Name == name }, true)
-		if r.Name == name {
+	if len(result) == 0 {
-			return r, nil
+		return nil, fmt.Errorf("mirror with name %s not found", name)
-		}
 	}
-	return nil, fmt.Errorf("mirror with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *RemoteRepoCollection) ByUUID(uuid string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
+	if r, ok := collection.cache[uuid]; ok {
-		if r.UUID == uuid {
+		return r, nil
-			return r, nil
-		}
 	}
-	return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+
+	key := (&RemoteRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	r := &RemoteRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &RemoteRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding mirror: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *RemoteRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("R")))
 }
 
 // Drop removes remote repo from collection
 func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
-	repoPosition := -1
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
 		panic("repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+	delete(collection.cache, repo.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/remote_test.go

@@ -88,8 +88,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.NewOpenDB(c.MkDir())
@@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 
 	s.repo.DownloadSources = true
 	s.repo.DownloadUdebs = true
+	s.repo.DownloadInstaller = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
 	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
+func (s *RemoteRepoSuite) TestInstallerPath(c *C) {
+	c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS")
+}
+
 func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
 	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
 }
@@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -333,6 +338,49 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(s.repo.packageRefs, NotNil)
 }
 
+func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
+	s.repo.Architectures = []string{"i386"}
+	s.repo.DownloadInstaller = true
+
+	err := s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/SHA256SUMS", exampleInstallerHashSumFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST", exampleInstallerManifestFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile)))
+	c.Check(queue, HasLen, 2)
+
+	q := make([]string, 2)
+	for i := range q {
+		q[i] = queue[i].File.DownloadURL()
+	}
+	sort.Strings(q)
+	c.Check(q[0], Equals, "dists/squeeze/main/installer-i386/current/images/MANIFEST")
+	c.Check(q[1], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
+	c.Assert(err, IsNil)
+	c.Check(pkg.Name, Equals, "installer")
+}
+
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.repo.Architectures = []string{"i386"}
 	s.repo.DownloadSources = true
@@ -347,7 +395,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -391,7 +439,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -416,7 +464,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -440,7 +488,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -468,7 +516,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -490,7 +538,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -521,7 +569,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -567,7 +615,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -593,7 +641,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -628,7 +676,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -646,16 +694,21 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewRemoteRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -676,7 +729,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -698,10 +751,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
@@ -874,4 +927,31 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
 `
 
+const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a  ./MANIFEST
+`
+
+const exampleInstallerManifestFile = `cdrom/debian-cd_info.tar.gz     -- isolinux config files for CD
+cdrom/gtk/debian-cd_info.tar.gz -- isolinux help screens for CD (graphical)
+cdrom/gtk/initrd.gz             -- initrd for use with isolinux to build a CD (graphical)
+cdrom/gtk/vmlinuz               -- kernel for use with isolinux to build a CD (graphical)
+cdrom/initrd.gz                 -- initrd for use with isolinux to build a CD
+cdrom/vmlinuz                   -- kernel for use with isolinux to build a CD
+cdrom/xen/debian.cfg            -- example Xen configuration
+cdrom/xen/initrd.gz             -- initrd for installing under Xen
+cdrom/xen/vmlinuz               -- kernel image for installing under Xen
+hd-media/boot.img.gz            -- 1 gb image (compressed) for USB memory stick
+hd-media/gtk/initrd.gz          -- for use on USB memory sticks
+hd-media/gtk/vmlinuz            -- for use on USB memory sticks
+hd-media/initrd.gz              -- for use on USB memory sticks
+hd-media/vmlinuz                -- for use on USB memory sticks
+netboot/debian-installer        -- PXE boot directory for tftp server
+netboot/gtk/debian-installer    -- PXE boot directory for tftp server (graphical installer)
+netboot/gtk/mini.iso            -- not so tiny CD image that boots the graphical netboot installer
+netboot/gtk/netboot.tar.gz      -- tarball of PXE boot directory (graphical installer)
+netboot/mini.iso                -- tiny CD image that boots the netboot installer
+netboot/netboot.tar.gz          -- tarball of PXE boot directory
+netboot/xen/debian.cfg          -- example Xen configuration
+netboot/xen/initrd.gz           -- initrd for installing under Xen
+netboot/xen/vmlinuz             -- kernel image for installing under Xen`
+
 const exampleSourcesFile = sourcePackageMeta

deb/snapshot.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -173,50 +173,36 @@ func (s *Snapshot) Decode(input []byte) error {
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
 type SnapshotCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*Snapshot
+	cache map[string]*Snapshot
 }
 
 // NewSnapshotCollection loads Snapshots from DB and makes up collection
 func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
-	result := &SnapshotCollection{
+	return &SnapshotCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   map[string]*Snapshot{},
 	}
-
-	blobs := db.FetchByPrefix([]byte("S"))
-	result.list = make([]*Snapshot, 0, len(blobs))
-
-	for _, blob := range blobs {
-		s := &Snapshot{}
-		if err := s.Decode(blob); err != nil {
-			log.Printf("Error decoding snapshot: %s\n", err)
-		} else {
-			result.list = append(result.list, s)
-		}
-	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *SnapshotCollection) Add(snapshot *Snapshot) error {
-	for _, s := range collection.list {
+	_, err := collection.ByName(snapshot.Name)
-		if s.Name == snapshot.Name {
+	if err == nil {
-			return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
+		return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
-		}
 	}
 
-	err := collection.Update(snapshot)
+	err = collection.Update(snapshot)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, snapshot)
+	collection.cache[snapshot.UUID] = snapshot
 	return nil
 }
 
-// Update stores updated information about repo in DB
+// Update stores updated information about snapshot in DB
 func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	err := collection.db.Put(snapshot.Key(), snapshot.Encode())
 	if err != nil {
@@ -239,83 +225,132 @@ func (collection *SnapshotCollection) LoadComplete(snapshot *Snapshot) error {
 	return snapshot.packageRefs.Decode(encoded)
 }
 
-// ByName looks up snapshot by name
+func (collection *SnapshotCollection) search(filter func(*Snapshot) bool, unique bool) []*Snapshot {
-func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
+	result := []*Snapshot(nil)
-	for _, s := range collection.list {
+	for _, s := range collection.cache {
-		if s.Name == name {
+		if filter(s) {
-			return s, nil
+			result = append(result, s)
 		}
 	}
+
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+			return nil
+		}
+
+		if filter(s) {
+			if _, exists := collection.cache[s.UUID]; !exists {
+				collection.cache[s.UUID] = s
+				result = append(result, s)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return result
+}
+
+// ByName looks up snapshot by name
+func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
+	result := collection.search(func(s *Snapshot) bool { return s.Name == name }, true)
+	if len(result) > 0 {
+		return result[0], nil
+	}
+
 	return nil, fmt.Errorf("snapshot with name %s not found", name)
 }
 
 // ByUUID looks up snapshot by UUID
 func (collection *SnapshotCollection) ByUUID(uuid string) (*Snapshot, error) {
-	for _, s := range collection.list {
+	if s, ok := collection.cache[uuid]; ok {
-		if s.UUID == uuid {
+		return s, nil
-			return s, nil
-		}
 	}
-	return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+
+	key := (&Snapshot{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Snapshot{}
+	err = s.Decode(value)
+
+	if err == nil {
+		collection.cache[s.UUID] = s
+	}
+
+	return s, err
 }
 
 // ByRemoteRepoSource looks up snapshots that have specified RemoteRepo as a source
 func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // ByLocalRepoSource looks up snapshots that have specified LocalRepo as a source
 func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // BySnapshotSource looks up snapshots that have specified snapshot as a source
 func (collection *SnapshotCollection) BySnapshotSource(snapshot *Snapshot) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
-	for _, s := range collection.list {
+		s := &Snapshot{}
-		err = handler(s)
+		if err := s.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(s)
+	})
 }
 
 // ForEachSorted runs method for each snapshot following some sort order
 func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error {
-	sorter, err := newSnapshotSorter(sortMethod, collection)
+	blobs := collection.db.FetchByPrefix([]byte("S"))
+	list := make([]*Snapshot, 0, len(blobs))
+
+	for _, blob := range blobs {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+		} else {
+			list = append(list, s)
+		}
+	}
+
+	sorter, err := newSnapshotSorter(sortMethod, list)
 	if err != nil {
 		return err
 	}
 
-	for _, i := range sorter.list {
+	for _, s := range sorter.list {
-		err = handler(collection.list[i])
+		err = handler(s)
 		if err != nil {
 			return err
 		}
@@ -327,26 +362,16 @@ func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler f
 // Len returns number of snapshots in collection
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("S")))
 }
 
 // Drop removes snapshot from collection
 func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
-	snapshotPosition := -1
+	if _, err := collection.db.Get(snapshot.Key()); err == database.ErrNotFound {
-
-	for i, s := range collection.list {
-		if s == snapshot {
-			snapshotPosition = i
-			break
-		}
-	}
-
-	if snapshotPosition == -1 {
 		panic("snapshot not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[snapshotPosition], collection.list =
+	delete(collection.cache, snapshot.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(snapshot.Key())
 	if err != nil {
@@ -363,13 +388,12 @@ const (
 )
 
 type snapshotSorter struct {
-	list       []int
+	list       []*Snapshot
-	collection *SnapshotCollection
 	sortMethod int
 }
 
-func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snapshotSorter, error) {
+func newSnapshotSorter(sortMethod string, list []*Snapshot) (*snapshotSorter, error) {
-	s := &snapshotSorter{collection: collection}
+	s := &snapshotSorter{list: list}
 
 	switch sortMethod {
 	case "time", "Time":
@@ -380,11 +404,6 @@ func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snap
 		return nil, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
 	}
 
-	s.list = make([]int, len(collection.list))
-	for i := range s.list {
-		s.list[i] = i
-	}
-
 	sort.Sort(s)
 
 	return s, nil
@@ -397,9 +416,9 @@ func (s *snapshotSorter) Swap(i, j int) {
 func (s *snapshotSorter) Less(i, j int) bool {
 	switch s.sortMethod {
 	case SortName:
-		return s.collection.list[s.list[i]].Name < s.collection.list[s.list[j]].Name
+		return s.list[i].Name < s.list[j].Name
 	case SortTime:
-		return s.collection.list[s.list[i]].CreatedAt.Before(s.collection.list[s.list[j]].CreatedAt)
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
 	}
 	panic("unknown sort method")
 }

deb/snapshot_bench_test.go

@@ -0,0 +1,98 @@
+package deb
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+func BenchmarkSnapshotCollectionForEach(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		collection.ForEach(func(s *Snapshot) error {
+			return nil
+		})
+	}
+}
+
+func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	uuids := []string{}
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+		uuids = append(uuids, snapshot.UUID)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		if _, err := collection.ByUUID(uuids[i%len(uuids)]); err != nil {
+			b.FailNow()
+		}
+	}
+}
+
+func BenchmarkSnapshotCollectionByName(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		if _, err := collection.ByName(fmt.Sprintf("snapshot%d", i%count)); err != nil {
+			b.FailNow()
+		}
+	}
+}

deb/snapshot_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"errors"
+	"sort"
 
 	"github.com/aptly-dev/aptly/database"
 
@@ -17,7 +18,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -116,11 +117,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -158,6 +159,10 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 	snapshot, err = collection.ByUUID(s.snapshot1.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.String(), Equals, s.snapshot1.String())
+
+	snapshot, err = collection.ByUUID(s.snapshot2.UUID)
+	c.Assert(err, IsNil)
+	c.Assert(snapshot.String(), Equals, s.snapshot2.String())
 }
 
 func (s *SnapshotCollectionSuite) TestUpdateLoadComplete(c *C) {
@@ -193,6 +198,23 @@ func (s *SnapshotCollectionSuite) TestForEachAndLen(c *C) {
 	c.Assert(err, Equals, e)
 }
 
+func (s *SnapshotCollectionSuite) TestForEachSorted(c *C) {
+	s.collection.Add(s.snapshot2)
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot4)
+	s.collection.Add(s.snapshot3)
+
+	names := []string{}
+
+	err := s.collection.ForEachSorted("name", func(snapshot *Snapshot) error {
+		names = append(names, snapshot.Name)
+		return nil
+	})
+	c.Assert(err, IsNil)
+
+	c.Check(sort.StringsAreSorted(names), Equals, true)
+}
+
 func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Assert(s.collection.Add(s.snapshot1), IsNil)
 	c.Assert(s.collection.Add(s.snapshot2), IsNil)
@@ -200,7 +222,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil))
 }
@@ -230,7 +252,11 @@ func (s *SnapshotCollectionSuite) TestFindSnapshotSource(c *C) {
 	c.Assert(s.collection.Add(snapshot4), IsNil)
 	c.Assert(s.collection.Add(snapshot5), IsNil)
 
-	c.Check(s.collection.BySnapshotSource(s.snapshot1), DeepEquals, []*Snapshot{snapshot3, snapshot4})
+	list := s.collection.BySnapshotSource(s.snapshot1)
+	sorter, _ := newSnapshotSorter("name", list)
+	sort.Sort(sorter)
+
+	c.Check(sorter.list, DeepEquals, []*Snapshot{snapshot3, snapshot4})
 	c.Check(s.collection.BySnapshotSource(s.snapshot2), DeepEquals, []*Snapshot{snapshot3})
 	c.Check(s.collection.BySnapshotSource(snapshot5), DeepEquals, []*Snapshot(nil))
 }

deb/testdata/changes/calamares.changes

@@ -0,0 +1,32 @@
+Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb

deb/testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes

@@ -0,0 +1,22 @@
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 invalidhardlink_0.2.1.dsc
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 invalidhardlink_0.2.1.dsc
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional invalidhardlink_0.2.1.dsc

deb/testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes

@@ -0,0 +1,39 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm1DLACgwW4V8qLQC/QHC/7+t3Iq47Ez
+eesAn3ZYLQvLYRw3wPTKVAPI+AW6Fjxi
+=hRBo
+-----END PGP SIGNATURE-----

deb/testdata/changes/hardlink_0.2.1.dsc

@@ -0,0 +1,19 @@
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: any
+Version: 0.2.1
+Maintainer: Julian Andres Klode <jak@debian.org>
+Homepage: http://jak-linux.org/projects/hardlink/
+Standards-Version: 3.9.3
+Vcs-Browser: http://git.debian.org/?p=users/jak/hardlink.git;a=summary
+Vcs-Git: git://git.debian.org/git/users/jak/hardlink.git
+Build-Depends: debhelper (>= 9), pkg-config, libpcre3-dev
+Package-List: 
+ hardlink deb utils optional
+Checksums-Sha1: 
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+Checksums-Sha256: 
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+Files: 
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 hardlink_0.2.1.tar.gz

deb/testdata/changes/hardlink_0.2.1_amd64.buildinfo

@@ -0,0 +1,170 @@
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: amd64
+Version: 0.2.0
+Checksums-Md5:
+ 2081e20b36c47f82811c25841cc0e41b 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha1: 
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Build-Origin: Debian
+Build-Architecture: amd64
+Build-Kernel-Version: 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07)
+Build-Date: Sun, 21 Jul 2019 08:01:03 +1400
+Build-Path: /build/hardlink-0.3.0/2nd
+Installed-Build-Depends:
+ autoconf (= 2.69-11),
+ automake (= 1:1.15.1-3.1),
+ autopoint (= 0.19.8.1-6),
+ autotools-dev (= 20180224.1),
+ base-files (= 10.1),
+ base-passwd (= 3.5.45),
+ bash (= 4.4.18-3),
+ binutils (= 2.30-21),
+ binutils-common (= 2.30-21),
+ binutils-x86-64-linux-gnu (= 2.30-21),
+ bsdmainutils (= 11.1.2+b1),
+ bsdutils (= 1:2.32-0.1),
+ build-essential (= 12.5),
+ bzip2 (= 1.0.6-8.1),
+ coreutils (= 8.28-1),
+ cpp (= 4:7.3.0-3),
+ cpp-7 (= 7.3.0-26+really21.0~reproducible0),
+ dash (= 0.5.8-2.10),
+ debconf (= 1.5.67),
+ debhelper (= 11.3.2),
+ debianutils (= 4.8.6),
+ dh-autoreconf (= 19),
+ dh-strip-nondeterminism (= 0.042-1),
+ diffutils (= 1:3.6-1),
+ dpkg (= 1.19.0.5.0~reproducible1),
+ dpkg-dev (= 1.19.0.5.0~reproducible1),
+ dwz (= 0.12-2),
+ fdisk (= 2.32-0.1),
+ file (= 1:5.33-3),
+ findutils (= 4.6.0+git+20171230-2),
+ g++ (= 4:7.3.0-3),
+ g++-7 (= 7.3.0-26+really21.0~reproducible0),
+ gcc (= 4:7.3.0-3),
+ gcc-7 (= 7.3.0-26+really21.0~reproducible0),
+ gcc-7-base (= 7.3.0-26+really21.0~reproducible0),
+ gcc-8-base (= 8.1.0-6),
+ gettext (= 0.19.8.1-6+b1),
+ gettext-base (= 0.19.8.1-6+b1),
+ grep (= 3.1-2),
+ groff-base (= 1.22.3-10),
+ gzip (= 1.6-5+b1),
+ hostname (= 3.20),
+ init-system-helpers (= 1.51),
+ intltool-debian (= 0.35.0+20060710.4),
+ libacl1 (= 2.2.52-3+b1),
+ libarchive-zip-perl (= 1.60-1),
+ libasan4 (= 7.3.0-26+really21.0~reproducible0),
+ libatomic1 (= 8.1.0-6),
+ libattr1 (= 1:2.4.47-2+b2),
+ libattr1-dev (= 1:2.4.47-2+b2),
+ libaudit-common (= 1:2.8.3-1),
+ libaudit1 (= 1:2.8.3-1),
+ libbinutils (= 2.30-21),
+ libblkid1 (= 2.32-0.1),
+ libbsd0 (= 0.9.1-1),
+ libbz2-1.0 (= 1.0.6-8.1),
+ libc-bin (= 2.27-3),
+ libc-dev-bin (= 2.27-3),
+ libc6 (= 2.27-3),
+ libc6-dev (= 2.27-3),
+ libcap-ng0 (= 0.7.9-1),
+ libcc1-0 (= 8.1.0-6),
+ libcilkrts5 (= 7.3.0-26+really21.0~reproducible0),
+ libcroco3 (= 0.6.12-2),
+ libdb5.3 (= 5.3.28-13.1+b1),
+ libdebconfclient0 (= 0.243),
+ libdpkg-perl (= 1.19.0.5.0~reproducible1),
+ libelf1 (= 0.170-0.4),
+ libfdisk1 (= 2.32-0.1),
+ libffi6 (= 3.2.1-8),
+ libfile-stripnondeterminism-perl (= 0.042-1),
+ libfreetype6 (= 2.8.1-2),
+ libgcc-7-dev (= 7.3.0-26+really21.0~reproducible0),
+ libgcc1 (= 1:8.1.0-6),
+ libgcrypt20 (= 1.8.3-1),
+ libgdbm-compat4 (= 1.14.1-6+b1),
+ libgdbm5 (= 1.14.1-6+b1),
+ libglib2.0-0 (= 2.56.1-2),
+ libgmp10 (= 2:6.1.2+dfsg-3),
+ libgomp1 (= 8.1.0-6),
+ libgpg-error0 (= 1.31-1),
+ libgraphite2-3 (= 1.3.11-2),
+ libharfbuzz0b (= 1.7.6-1+b1),
+ libicu-le-hb0 (= 1.0.3+git161113-5),
+ libicu60 (= 60.2-6),
+ libisl19 (= 0.19-1),
+ libitm1 (= 8.1.0-6),
+ liblsan0 (= 8.1.0-6),
+ liblz4-1 (= 1.8.2-1),
+ liblzma5 (= 5.2.2-1.3),
+ libmagic-mgc (= 1:5.33-3),
+ libmagic1 (= 1:5.33-3),
+ libmount1 (= 2.32-0.1),
+ libmpc3 (= 1.1.0-1),
+ libmpfr6 (= 4.0.1-1),
+ libmpx2 (= 8.1.0-6),
+ libncurses6 (= 6.1+20180210-4),
+ libncursesw6 (= 6.1+20180210-4),
+ libpam-modules (= 1.1.8-3.7),
+ libpam-modules-bin (= 1.1.8-3.7),
+ libpam-runtime (= 1.1.8-3.7),
+ libpam0g (= 1.1.8-3.7),
+ libpcre16-3 (= 2:8.39-9),
+ libpcre3 (= 2:8.39-9),
+ libpcre3-dev (= 2:8.39-9),
+ libpcre32-3 (= 2:8.39-9),
+ libpcrecpp0v5 (= 2:8.39-9),
+ libperl5.26 (= 5.26.2-6),
+ libpipeline1 (= 1.5.0-1),
+ libpng16-16 (= 1.6.34-1),
+ libquadmath0 (= 8.1.0-6),
+ libseccomp2 (= 2.3.3-2),
+ libselinux1 (= 2.8-1),
+ libsigsegv2 (= 2.12-2),
+ libsmartcols1 (= 2.32-0.1),
+ libstdc++-7-dev (= 7.3.0-26+really21.0~reproducible0),
+ libstdc++6 (= 8.1.0-6),
+ libsystemd0 (= 238-5),
+ libtimedate-perl (= 2.3000-2),
+ libtinfo6 (= 6.1+20180210-4),
+ libtool (= 2.4.6-2.1),
+ libtsan0 (= 8.1.0-6),
+ libubsan0 (= 7.3.0-26+really21.0~reproducible0),
+ libudev1 (= 238-5),
+ libunistring2 (= 0.9.8-1),
+ libuuid1 (= 2.32-0.1),
+ libxml2 (= 2.9.4+dfsg1-7),
+ linux-libc-dev (= 4.16.12-1),
+ login (= 1:4.5-1),
+ m4 (= 1.4.18-1),
+ make (= 4.2.1-1),
+ man-db (= 2.8.3-2),
+ mawk (= 1.3.3-17+b3),
+ ncurses-base (= 6.1+20180210-4),
+ ncurses-bin (= 6.1+20180210-4),
+ patch (= 2.7.6-2),
+ perl (= 5.26.2-6),
+ perl-base (= 5.26.2-6),
+ perl-modules-5.26 (= 5.26.2-6),
+ pkg-config (= 0.29-4+b1),
+ po-debconf (= 1.0.20),
+ sed (= 4.4-2),
+ sysvinit-utils (= 2.88dsf-59.10),
+ tar (= 1.30+dfsg-2),
+ util-linux (= 2.32-0.1),
+ xz-utils (= 5.2.2-1.3),
+ zlib1g (= 1:1.2.11.dfsg-1)
+Environment:
+ BUILD_PATH_PREFIX_MAP="hardlink_0.3.0=/build/hardlink-0.3.0/2nd"
+ DEB_BUILD_OPTIONS="buildinfo=+all parallel=16"
+ LANG="C"
+ LC_ALL="C"
+ SOURCE_DATE_EPOCH="1411647982"

deb/testdata/changes/hardlink_0.2.1_amd64.changes

@@ -0,0 +1,34 @@
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ d20d6820bfccf5e3a5b120eedabba96e71da60ff 703 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+ 06c38a55e81907e573641d7f979f428583bd3165 4885 hardlink_0.2.1_amd64.buildinfo
+ be2b66e32b29ab654190e7ecb4cc31f48ec061cb 12084 hardlink_0.2.0_i386.deb
+Checksums-Sha256: 
+ 19d08cfadd58aee05fd06aca261a873bafeb55cac58bf4916cc070f5821803e7 703 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+ 2a4905626b8664868b712a5b3d21f44c2665d17f9a4b67dc2e2bfd03f4ec54ee 4885 hardlink_0.2.1_amd64.buildinfo
+ a9d657d9413029e484c2a8c059c95a3e31eb1b47eca96455c1b9c330352293c9 12084 hardlink_0.2.0_i386.deb
+Files: 
+ b75fe0616f24deb28a3017c1dbae219a 703 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+ 9468a19700d038fa6cd10c5e42063083 12084 utils optional hardlink_0.2.0_i386.deb
+ 490174686848eca6da4e5f79d899efe7 4885 utils optional hardlink_0.2.1_amd64.buildinfo

deb/version_test.go

@@ -5,7 +5,6 @@ import (
 )
 
 type VersionSuite struct {
-	stanza Stanza
 }
 
 var _ = Suite(&VersionSuite{})

files/package_pool.go

@@ -9,7 +9,7 @@ import (
 	"sync"
 	"syscall"
 
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"

files/public.go

@@ -123,10 +123,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is a relative path to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
+	baseName := filepath.Base(fileName)
+	poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName))
 
 	err := os.MkdirAll(poolPath, 0777)
 	if err != nil {

files/public_test.go

@@ -181,39 +181,40 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 
 func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	tests := []struct {
-		prefix           string
+		prefix             string
-		component        string
+		sourcePath         string
-		sourcePath       string
+		publishedDirectory string
-		poolDirectory    string
+		expectedFilename   string
-		expectedFilename string
 	}{
 		{ // package name regular
-			prefix:           "",
+			prefix:             "",
-			component:        "main",
+			sourcePath:         "mars-invaders_1.03.deb",
-			sourcePath:       "mars-invaders_1.03.deb",
+			publishedDirectory: "pool/main/m/mars-invaders",
-			poolDirectory:    "m/mars-invaders",
+			expectedFilename:   "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
-			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // lib-like filename
-			prefix:           "",
+			prefix:             "",
-			component:        "main",
+			sourcePath:         "libmars-invaders_1.03.deb",
-			sourcePath:       "libmars-invaders_1.03.deb",
+			publishedDirectory: "pool/main/libm/libmars-invaders",
-			poolDirectory:    "libm/libmars-invaders",
+			expectedFilename:   "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
-			expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
 		},
 		{ // duplicate link, shouldn't panic
-			prefix:           "",
+			prefix:             "",
-			component:        "main",
+			sourcePath:         "mars-invaders_1.03.deb",
-			sourcePath:       "mars-invaders_1.03.deb",
+			publishedDirectory: "pool/main/m/mars-invaders",
-			poolDirectory:    "m/mars-invaders",
+			expectedFilename:   "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
-			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // prefix & component
-			prefix:           "ppa",
+			prefix:             "ppa",
-			component:        "contrib",
+			sourcePath:         "libmars-invaders_1.04.deb",
-			sourcePath:       "libmars-invaders_1.04.deb",
+			publishedDirectory: "pool/contrib/libm/libmars-invaders",
-			poolDirectory:    "libm/libmars-invaders",
+			expectedFilename:   "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
-			expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
+		},
+		{ // installer file
+			prefix:             "",
+			sourcePath:         "netboot/boot.img.gz",
+			publishedDirectory: "dists/jessie/non-free/installer-i386/current/images",
+			expectedFilename:   "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz",
 		},
 	}
 
@@ -221,6 +222,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 
 	for _, t := range tests {
 		tmpPath := filepath.Join(c.MkDir(), t.sourcePath)
+		os.MkdirAll(filepath.Dir(tmpPath), 0777)
 		err := ioutil.WriteFile(tmpPath, []byte("Contents"), 0644)
 		c.Assert(err, IsNil)
 
@@ -231,7 +233,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Assert(err, IsNil)
 
 		// Test using hardlinks
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
@@ -241,7 +243,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Nlink), Equals, 3)
 
 		// Test using symlinks
-		err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
@@ -252,7 +254,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
 
 		// Test using copy with checksum verification
-		err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
@@ -262,7 +264,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		c.Check(int(info.Nlink), Equals, 1)
 
 		// Test using copy with size verification
-		err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, t.publishedDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))

http/compression.go

@@ -4,7 +4,6 @@ import (
 	"compress/bzip2"
 	"compress/gzip"
 	"context"
-	"fmt"
 	"io"
 	"net/url"
 	"os"
@@ -90,7 +89,7 @@ func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, ba
 	}
 
 	if err == nil {
-		err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path}))
+		return nil, nil, &NoCandidateFoundError{URL: baseURL.ResolveReference(&url.URL{Path: path})}
 	}
 	return nil, nil, err
 }

http/download.go

@@ -64,6 +64,29 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 	return downloader.progress
 }
 
+// GetLength of given url
+func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (int64, error) {
+	req, err := downloader.newRequest(ctx, "HEAD", url)
+	if err != nil {
+		return -1, err
+	}
+
+	resp, err := downloader.client.Do(req)
+	if err != nil {
+		return -1, errors.Wrap(err, url)
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return -1, &Error{Code: resp.StatusCode, URL: url}
+	}
+
+	if resp.ContentLength < 0 {
+		return -1, fmt.Errorf("could not determine length of %s", url)
+	}
+
+	return resp.ContentLength, nil
+}
+
 // Download starts new download task
 func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
 	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
@@ -71,25 +94,20 @@ func (downloader *downloaderImpl) Download(ctx context.Context, url string, dest
 
 func retryableError(err error) bool {
 	switch err.(type) {
-	case net.Error:
-		return true
 	case *net.OpError:
 		return true
 	case syscall.Errno:
 		return true
+	case net.Error:
+		return true
 	}
 	return false
 }
 
-// DownloadWithChecksum starts new download task with checksum verification
+func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url string) (*http.Request, error) {
-func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
+	req, err := http.NewRequest(method, url, nil)
-	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
-
-	downloader.progress.Printf("Downloading %s...\n", url)
-
-	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		return errors.Wrap(err, url)
+		return nil, errors.Wrap(err, url)
 	}
 	req.Close = true
 	req = req.WithContext(ctx)
@@ -100,6 +118,18 @@ func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url
 		req.URL.RawQuery = ""
 	}
 
+	return req, nil
+}
+
+// DownloadWithChecksum starts new download task with checksum verification
+func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
+	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+
+	if downloader.progress != nil {
+		downloader.progress.Printf("Downloading %s...\n", url)
+	}
+	req, err := downloader.newRequest(ctx, "GET", url)
+
 	var temppath string
 	for maxTries > 0 {
 		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)

http/download_test.go

@@ -123,3 +123,22 @@ func (s *DownloaderSuite) TestDownloadFileError(c *C) {
 	c.Assert(s.d.Download(s.ctx, s.url+"/test", "/"),
 		ErrorMatches, ".*permission denied")
 }
+
+func (s *DownloaderSuite) TestGetLength(c *C) {
+	size, err := s.d.GetLength(s.ctx, s.url+"/test")
+
+	c.Assert(err, IsNil)
+	c.Assert(size, Equals, int64(12))
+}
+
+func (s *DownloaderSuite) TestGetLength404(c *C) {
+	_, err := s.d.GetLength(s.ctx, s.url+"/doesntexist")
+
+	c.Assert(err, ErrorMatches, "HTTP code 404.*")
+}
+
+func (s *DownloaderSuite) TestGetLengthConnectError(c *C) {
+	_, err := s.d.GetLength(s.ctx, "http://nosuch.localhost/")
+
+	c.Assert(err, ErrorMatches, ".*no such host")
+}

http/fake.go

@@ -60,8 +60,17 @@ func (f *FakeDownloader) Empty() bool {
 	return len(f.expected) == 0
 }
 
-// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
+// GetLength returns content length of given url
-func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+func (f *FakeDownloader) GetLength(ctx context.Context, url string) (int64, error) {
+	expectation, err := f.getExpectedRequest(url)
+	if err != nil {
+		return -1, err
+	}
+
+	return int64(len(expectation.Response)), nil
+}
+
+func (f *FakeDownloader) getExpectedRequest(url string) (*expectedRequest, error) {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -69,14 +78,24 @@ func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, f
 		expectation = f.anyExpected[url]
 		delete(f.anyExpected, url)
 	} else {
-		return fmt.Errorf("unexpected request for %s", url)
+		return nil, fmt.Errorf("unexpected request for %s", url)
 	}
 
 	if expectation.Err != nil {
-		return expectation.Err
+		return nil, expectation.Err
 	}
 
-	err := os.MkdirAll(filepath.Dir(filename), 0755)
+	return &expectation, nil
+}
+
+// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
+func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
+	expectation, err := f.getExpectedRequest(url)
+	if err != nil {
+		return err
+	}
+
+	err = os.MkdirAll(filepath.Dir(filename), 0755)
 	if err != nil {
 		return err
 	}

http/http.go

@@ -3,6 +3,7 @@ package http
 
 import (
 	"fmt"
+	"net/url"
 )
 
 // Error is download error connected to HTTP code
@@ -15,3 +16,13 @@ type Error struct {
 func (e *Error) Error() string {
 	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
 }
+
+// NoCandidateFoundError indicates that now candidate of given url could be found
+type NoCandidateFoundError struct {
+	URL *url.URL
+}
+
+// Error message
+func (e *NoCandidateFoundError) Error() string {
+	return fmt.Sprintf("no candidates for %s found", e.URL)
+}

linter.json

@@ -1,24 +0,0 @@
-{
-    "DisableAll": true,
-    "Enable": [
-        "vet",
-        "golint",
-        "gofmt",
-        "deadcode",
-        "goimports",
-        "misspell",
-        "gosimple",
-        "ineffassign",
-        "staticcheck",
-        "varcheck",
-        "structcheck",
-        "maligned",
-        "vetshadow",
-        "goconst",
-        "interfacer"
-    ],
-    "Deadline": "20m",
-    "Vendor": true,
-    "VendoredLinters": true,
-    "Concurrency": 1
-}

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "November 2017" "" ""
+.TH "APTLY" "1" "September 2018" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -150,7 +150,7 @@ don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror
 .
 .TP
 \fBgpgProvider\fR
-implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation
+implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation; \fBgpg1\fR might be used to force use of GnuPG 1\.x, \fBgpg2\fR enables GnuPG 2\.x only; default is to use GnuPG 1\.x if available and GnuPG 2\.x otherwise
 .
 .TP
 \fBdownloadSourcePackages\fR
@@ -434,7 +434,7 @@ when processing dependencies, print detailed logs
 .
 .TP
 \-\fBgpg\-provider\fR=
-PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
 .
 .SH "CREATE NEW MIRROR"
 \fBaptly\fR \fBmirror\fR \fBcreate\fR \fIname\fR \fIarchive url\fR \fIdistribution\fR [\fIcomponent1\fR \|\.\|\.\|\.]
@@ -482,6 +482,10 @@ disable verification of Release file signatures
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
 .TP
+\-\fBwith\-installer\fR
+download additional not packaged installer files
+.
+.TP
 \-\fBwith\-sources\fR
 download source packages in addition to binary packages
 .
@@ -637,6 +641,10 @@ disable verification of Release file signatures
 gpg keyring to use when verifying Release file (could be specified multiple times)
 .
 .TP
+\-\fBwith\-installer\fR
+download additional not packaged installer files
+.
+.TP
 \-\fBwith\-sources\fR
 download source packages in addition to binary packages
 .
@@ -2030,5 +2038,17 @@ Matt Martyn (https://github\.com/MMartyn)
 .IP "\[ci]" 4
 Ludovico Cavedon (https://github\.com/cavedon)
 .
+.IP "\[ci]" 4
+Petr Jediny (https://github\.com/pjediny)
+.
+.IP "\[ci]" 4
+Maximilian Stein (https://github\.com/steinymity)
+.
+.IP "\[ci]" 4
+Strajan Sebastian (https://github\.com/strajansebastian)
+.
+.IP "\[ci]" 4
+Artem Smirnov (https://github\.com/urpylka)
+.
 .IP "" 0
 

man/aptly.1.ronn.tmpl

@@ -130,7 +130,9 @@ Options:
 
   * `gpgProvider`:
     implementation of PGP signing/validation - `gpg` for external `gpg` utility or
-    `internal` to use Go internal implementation
+    `internal` to use Go internal implementation; `gpg1` might be used to force use
+    of GnuPG 1.x, `gpg2` enables GnuPG 2.x only; default is to use GnuPG 1.x if
+    available and GnuPG 2.x otherwise
 
   * `downloadSourcePackages`:
     if enabled, all mirrors created would have flag set to download source packages;

pgp/4.text

@@ -0,0 +1,109 @@
+Origin: Canonical
+Suite: xenial-updates
+Codename: xenial-updates/newton
+Date: Tue, 25 Sep 2018 21:45:15 UTC
+Architectures: i386 amd64 armhf ppc64el arm64 s390x
+Components: main
+Description: OpenStack Newton archive for Ubuntu 16.04 LTS
+MD5Sum:
+ ed32606ac5a1ce86befae2715ebd5328 559539 main/binary-i386/Packages
+ 100b57acc19e7fc74994fa7f843a3fe6 120571 main/binary-i386/Packages.gz
+ 30687c5e34fd9eac9feb3e0ba4a0f94b 136 main/binary-i386/Release
+ 24e1b1243547d40ada70bb45343f4a56 560520 main/binary-amd64/Packages
+ 52aa3aab02dbf701572e02a43f0140b3 120716 main/binary-amd64/Packages.gz
+ beca9426b77f6322e4b9793b30f42183 137 main/binary-amd64/Release
+ 3fca39e42300b53d02e7453e4fed96a2 497267 main/binary-armhf/Packages
+ 8bf44c480558f5e922263cbc2c1394a5 108076 main/binary-armhf/Packages.gz
+ 5c38f748178697764f9724828f0151ab 137 main/binary-armhf/Release
+ 51cdb3afb5b2fcb5fc7f6fce22085628 516159 main/binary-ppc64el/Packages
+ 8d3bd39c70b096496d7d01ba9a525851 112620 main/binary-ppc64el/Packages.gz
+ 51dda1f860edc7975a3852e7c4f9f69d 139 main/binary-ppc64el/Release
+ 23471ddea998006fbf82825dc2d22f4e 557502 main/binary-arm64/Packages
+ 52d0175877152059aeec77114265df7e 120123 main/binary-arm64/Packages.gz
+ 6a909d0126b1e910fcd2b3f58d53ef7b 137 main/binary-arm64/Release
+ 89171df2220477022ef91e74c236d74d 516061 main/binary-s390x/Packages
+ bbfea425a605c53e59c440adc7d491c0 112610 main/binary-s390x/Packages.gz
+ eaa188bf3d6e821f73b7c29f9085973f 137 main/binary-s390x/Release
+ 9b21f7c2c0d89f08cd54d21526b2b7f7 417889 main/source/Sources
+ 8113c5d6484daf4f632cacd048aa81a7 81794 main/source/Sources.gz
+ 36c2b37c73e51cb58ec2769baaa237f2 138 main/source/Release
+ cce4a93beb7346dfcfc401343c1a07e7 5194695 Contents-i386
+ 099e00a2232232f0b891ab2924796cd4 321384 Contents-i386.gz
+ 148c884428ab15f5c16d4a8df0358f41 5195719 Contents-amd64
+ b7e9f1fad4d5e63f9ccaaf826dba4af5 321473 Contents-amd64.gz
+ 82f8be9e623dc1ac2191f04db01b2080 5121293 Contents-armhf
+ 933cb600a7d31d87ee2f862464ca0a71 313824 Contents-armhf.gz
+ 5d5dc264638d1b585499267e07793d57 5157787 Contents-ppc64el
+ da5b35ec8d94e9f7db3d6a76f36fa099 317603 Contents-ppc64el.gz
+ 7d2ad5ec50c13c92d7690567b3fc674a 5194665 Contents-arm64
+ bf3c7c2cd0d8ae68b670e85bf4c0a55f 321315 Contents-arm64.gz
+ 6fe4eb1eb6bc4ea36f393b43b30957d4 5157667 Contents-s390x
+ c69b1e4ca2d8ef58355f422bf282c173 317598 Contents-s390x.gz
+SHA1:
+ 7830f7d6f20f591dd8d268a4be4c7683c9fcbaf1 559539 main/binary-i386/Packages
+ eafced4e96ccf644cde26a7e16606c3f4079e252 120571 main/binary-i386/Packages.gz
+ 9aee52218b3cfa1df5f326ed2c2e562de255cb68 136 main/binary-i386/Release
+ a2ec30f4af65fffc8e3c57e0b3aa7771c2148f7b 560520 main/binary-amd64/Packages
+ 2d6afc82b4afe15a72e7bc79ff8b4096ea7f852d 120716 main/binary-amd64/Packages.gz
+ b8ee5a9cc3b3b73cd0844591eb7108d233e67991 137 main/binary-amd64/Release
+ 37edbb5717e48dd5ef4410e2cdaafad010dfb3f5 497267 main/binary-armhf/Packages
+ 166cc92eba00d6c7e8a73869fb0b4682ed401402 108076 main/binary-armhf/Packages.gz
+ 1e9009c47ba57153c77b58dc85fa80c6b7736c71 137 main/binary-armhf/Release
+ 3fed2e76b941dcc7177f858794ef5c26e6054ae5 516159 main/binary-ppc64el/Packages
+ 9b447bdbe3c8872a85e79c42cedd367dd78c1c22 112620 main/binary-ppc64el/Packages.gz
+ 2ff8e4116672ef13f0488240ffbe9893ef72d251 139 main/binary-ppc64el/Release
+ 175b7d0f77ec52b43769133f8a7c2568c573604a 557502 main/binary-arm64/Packages
+ 31dbc9776ec0f09902e403ab397e23abf8a27c6f 120123 main/binary-arm64/Packages.gz
+ 1d86c1ffd8bbaff5da76f9fd6e66df0b09885f69 137 main/binary-arm64/Release
+ b66894b8555a564d48808c757069d89d5e25749f 516061 main/binary-s390x/Packages
+ d76c22241faada5571975f3d7c1689e35456d1fd 112610 main/binary-s390x/Packages.gz
+ 927c8b132b4f41440c3342ff6dafa889b8debbf1 137 main/binary-s390x/Release
+ 0fa2981518816d511b8719c3ee28d309480cc97e 417889 main/source/Sources
+ 1de78aa62f50854c87b1ea56ee0fd1dd0485f9d7 81794 main/source/Sources.gz
+ 8dcbec8bfd3a98b8d9106f0a86e8583403d78458 138 main/source/Release
+ de31496319ebb8a06e843e2a855323ee996b11a8 5194695 Contents-i386
+ a85e1fb48738be3702bb2e0a7bfaec5ddda4a36e 321384 Contents-i386.gz
+ 4af01067070fe5206a5212a482ed221c0ab58553 5195719 Contents-amd64
+ fac22f57752875f95dce9ae9bce4f5919732b522 321473 Contents-amd64.gz
+ 7408f28ea1e5ef995b5545a9c5d31f5a06f1deed 5121293 Contents-armhf
+ dfdf1eee9652d079d7f00d102c6a9d69f76d2134 313824 Contents-armhf.gz
+ c348534f065afc7bf171eeb64da87417fb8ff3b1 5157787 Contents-ppc64el
+ a843a8b2f7a7c453c1c8aaa349bf5f95c4391692 317603 Contents-ppc64el.gz
+ ac080e8fb26c2726ce14b2b89e847c3a46953276 5194665 Contents-arm64
+ 0d94c3e512eb46891799a922f4db41dbd8f7a677 321315 Contents-arm64.gz
+ 5dbb522f499ee5d316346cf135e1d0eaba0a8ee2 5157667 Contents-s390x
+ 1039320511a46c2232b53113c88eb47f19fc49db 317598 Contents-s390x.gz
+SHA256:
+ 2ae20b2535fdb2a8f2e4b51a139c45ad5effc1601db5910074030d1b240bf6d8 559539 main/binary-i386/Packages
+ e297fe58c023868db53045d3743c879189d31bf756320bedbe9aeed46d2a9b18 120571 main/binary-i386/Packages.gz
+ d225d69dc14ad076639111f91c52d2a3dd69bb1a92c3bedc60c82486f6199634 136 main/binary-i386/Release
+ 1a00ac12805e0def30228d5483373e24ffe1b5b77543624c75652ebeb25f9945 560520 main/binary-amd64/Packages
+ a622da857847f178dc554909597c522f5cdd7f953f2067a1493a806f3e43b134 120716 main/binary-amd64/Packages.gz
+ cd1a07c3615a54e20f47d7ccd8e73ceaa5a371571a699dd5e8ae11d9066f9929 137 main/binary-amd64/Release
+ 25be54171d77d9732aa234ed1505f82fce44ab90fc223440708f6e86696d8a45 497267 main/binary-armhf/Packages
+ 9dc79733217cbf28b9b92684c9d37ca5011fae6b20fddc67b2117d7551915faa 108076 main/binary-armhf/Packages.gz
+ d86d50eb4a75ce2817ce5a6a7d792e822e105a139c5fb279fbfadfd96c9d9fc1 137 main/binary-armhf/Release
+ e5c488274bb806394f0a4a253a41609cfd888754e1dbc184e308fd8f56d50795 516159 main/binary-ppc64el/Packages
+ 4001d9cf9223c876a31cf8d7adc0c1144630a82d1e65e595cdd5bca9222559dd 112620 main/binary-ppc64el/Packages.gz
+ 5b2da9b67840f666a54164f2a6e8333e5977ddc362af188bda5e99eb96bb3fde 139 main/binary-ppc64el/Release
+ 332d7c702c4ef4b3aa8178852945383829db6cdfa4d7c5b4bebea3016244ce10 557502 main/binary-arm64/Packages
+ cc25d0fc96f92d894ed765bf4ec25330c88b3d1f4395c107dfa720bf719a5f2b 120123 main/binary-arm64/Packages.gz
+ 3deb687a4f3c3ac0a24733e90578c0bcfb4f3cd5e692dce0c5d293d76bca073c 137 main/binary-arm64/Release
+ cb01f4e6501350005260311d818444df6a8c8c76ef29d145b2bf7031fa808f1c 516061 main/binary-s390x/Packages
+ 95f7f94b492c83992efad814ea228c9cdd1b04347b4118532f0dd1b83cf66351 112610 main/binary-s390x/Packages.gz
+ 154ed5c2c7356d6ae63340ed5e8f73806f82af958510084ccb3e36c19fc39d0b 137 main/binary-s390x/Release
+ 68b03eed15470eaf8df0702abe623a6ec055a117851050eec50a203d44f6ff25 417889 main/source/Sources
+ 770e405fb223116b02a5b71df93b55db037703126044b45338b8d03c2ab5f9e6 81794 main/source/Sources.gz
+ 32aa4fbf5c619b3491df7dbfc98a752cf8235d770775c9207ebb6bfa0b236c16 138 main/source/Release
+ f03799e6858a078dafaab8740acafc9184b7d1a070ec94ff2c0589771175a2bb 5194695 Contents-i386
+ 4e546e8e37208d2433dfcdfe8de707bb8062ddb0b80e92696d43169f81bb326d 321384 Contents-i386.gz
+ 5d155efdccd8258b493f039bf55d038630465bf064301e115198dcae52dd6065 5195719 Contents-amd64
+ c66c2b872aca23d1595902b1091d697e7d116d9580d5e255838e99a439789845 321473 Contents-amd64.gz
+ cc60e26e7fd2c2f9efc2060ce981b02a396941b5d028629a1d4ea85f768bb59d 5121293 Contents-armhf
+ 6f5a036d3f471bc9f9912ae2531641d69464174565eb7be0661eccf5cb023762 313824 Contents-armhf.gz
+ d0c9b1ce2d5e0dbc3f83bf63edb8369d6c6958fd653999a00064ce7aa387540a 5157787 Contents-ppc64el
+ 235241b688107b2b40c55dbb883e051754bffaaa7eaccf9da762ffa6da429570 317603 Contents-ppc64el.gz
+ 74f39e2423ce5bb033b824e4e3b642ac19db472d5cc2a8faa4dad49932b97211 5194665 Contents-arm64
+ 3829445563cd1b75f79dd86756437efe7a6eb0a2601e9e423faf7bb8e9ad2aa7 321315 Contents-arm64.gz
+ 6280906ac14ccc3e9947e018a3296bfa1e4081df75a4fe2a78f92bb93418d706 5157667 Contents-s390x
+ 86180bba76cdbebe566f0075680a99b4e19c4308e302e16d5d95a2008b255d17 317598 Contents-s390x.gz

pgp/gnupg.go

@@ -3,6 +3,7 @@ package pgp
 import (
 	"bufio"
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -21,6 +22,7 @@ var (
 // GpgSigner is implementation of Signer interface using gpg as external program
 type GpgSigner struct {
 	gpg                        string
+	version                    GPGVersion
 	keyRef                     string
 	keyring, secretKeyring     string
 	passphrase, passphraseFile string
@@ -52,7 +54,7 @@ func (g *GpgSigner) gpgArgs() []string {
 	if g.keyring != "" {
 		args = append(args, "--no-auto-check-trustdb", "--no-default-keyring", "--keyring", g.keyring)
 	}
-	if g.secretKeyring != "" {
+	if g.secretKeyring != "" && g.version == GPG1x {
 		args = append(args, "--secret-keyring", g.secretKeyring)
 	}
 
@@ -61,7 +63,9 @@ func (g *GpgSigner) gpgArgs() []string {
 	}
 
 	if g.passphrase != "" || g.passphraseFile != "" {
-		args = append(args, "--no-use-agent")
+		if g.version == GPG1x {
+			args = append(args, "--no-use-agent")
+		}
 	}
 
 	if g.passphrase != "" {
@@ -74,53 +78,21 @@ func (g *GpgSigner) gpgArgs() []string {
 
 	if g.batch {
 		args = append(args, "--no-tty", "--batch")
+		if g.version == GPG21xPlus {
+			args = append(args, "--pinentry-mode", "loopback")
+		}
 	}
 
 	return args
 }
 
-func cliVersionCheck(cmd string, marker string) bool {
-	output, err := exec.Command(cmd, "--version").CombinedOutput()
-	if err != nil {
-		return false
-	}
-	return strings.Contains(string(output), marker)
-}
-
-func findSuitableCLI(cmds []string, versionMarker string) string {
-	for _, cmd := range cmds {
-		if cliVersionCheck(cmd, versionMarker) {
-			return cmd
-		}
-	}
-	return ""
-}
-
-// We only support gpg1 at this time. Make sure we find a suitable binary.
-func findGPG1() (string, error) {
-	cmd := findSuitableCLI([]string{"gpg", "gpg1"}, "gpg (GnuPG) 1.")
-	if cmd != "" {
-		return cmd, nil
-	}
-	return "", fmt.Errorf("Couldn't find a suitable gpg executable. Make sure gnupg1 is available as either gpg or gpg1 in $PATH")
-}
-
-// We only support gpgv1 at this time. Make sure we find a suitable binary.
-func findGPGV1() (string, error) {
-	cmd := findSuitableCLI([]string{"gpgv", "gpgv1"}, "gpgv (GnuPG) 1.")
-	if cmd != "" {
-		return cmd, nil
-	}
-	return "", fmt.Errorf("Couldn't find a suitable gpgv executable. Make sure gpgv1 is available as either gpgv or gpgv1 in $PATH")
-}
-
 // NewGpgSigner creates a new gpg signer
-func NewGpgSigner() *GpgSigner {
+func NewGpgSigner(finder GPGFinder) *GpgSigner {
-	gpg, err := findGPG1()
+	gpg, version, err := finder.FindGPG()
 	if err != nil {
 		panic(err)
 	}
-	return &GpgSigner{gpg: gpg}
+	return &GpgSigner{gpg: gpg, version: version}
 }
 
 // Init verifies availability of gpg & presence of keys
@@ -168,38 +140,31 @@ func (g *GpgSigner) ClearSign(source string, destination string) error {
 type GpgVerifier struct {
 	gpg      string
 	gpgv     string
+	version  GPGVersion
 	keyRings []string
 }
 
-// NewGpgVerifier creates a new gpg signer
+// NewGpgVerifier creates a new gpg verifier
-func NewGpgVerifier() *GpgVerifier {
+func NewGpgVerifier(finder GPGFinder) *GpgVerifier {
-	gpg, err := findGPG1()
+	gpg, versionGPG, err := finder.FindGPG()
 	if err != nil {
 		panic(err)
 	}
 
-	gpgv, err := findGPGV1()
+	gpgv, versionGPGV, err := finder.FindGPGV()
 	if err != nil {
 		panic(err)
 	}
 
-	return &GpgVerifier{gpg: gpg, gpgv: gpgv}
+	if versionGPG != versionGPGV {
+		panic(errors.New("gpg and gpgv versions don't match"))
+	}
+
+	return &GpgVerifier{gpg: gpg, gpgv: gpgv, version: versionGPG}
 }
 
 // InitKeyring verifies that gpg is installed and some keys are trusted
 func (g *GpgVerifier) InitKeyring() error {
-	cmd, err := findGPG1()
-	if err != nil {
-		return err
-	}
-	g.gpg = cmd
-
-	cmd, err = findGPGV1()
-	if err != nil {
-		return err
-	}
-	g.gpgv = cmd
-
 	if len(g.keyRings) == 0 {
 		// using default keyring
 		output, err := exec.Command(g.gpg, "--no-default-keyring", "--no-auto-check-trustdb", "--keyring", "trustedkeys.gpg", "--list-keys").Output()

pgp/gnupg_finder.go

@@ -0,0 +1,149 @@
+package pgp
+
+import (
+	"errors"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+// GPGVersion stores discovered GPG version
+type GPGVersion int
+
+// GPG version as discovered
+const (
+	GPG1x      GPGVersion = 1
+	GPG20x     GPGVersion = 2
+	GPG21xPlus GPGVersion = 3
+)
+
+var gpgVersionRegex = regexp.MustCompile(`\(GnuPG\) (\d)\.(\d)`)
+
+// GPGFinder implement search for gpg executables and returns version of discovered executables
+type GPGFinder interface {
+	FindGPG() (gpg string, version GPGVersion, err error)
+	FindGPGV() (gpgv string, version GPGVersion, err error)
+}
+
+type pathGPGFinder struct {
+	gpgNames     []string
+	gpgvNames    []string
+	errorMessage string
+
+	expectedVersionSubstring string
+}
+
+type iteratingGPGFinder struct {
+	finders      []GPGFinder
+	errorMessage string
+}
+
+// GPGDefaultFinder looks for GPG1 first, but falls back to GPG2 if GPG1 is not available
+func GPGDefaultFinder() GPGFinder {
+	return &iteratingGPGFinder{
+		finders:      []GPGFinder{GPG1Finder(), GPG2Finder()},
+		errorMessage: "Couldn't find a suitable gpg executable. Make sure gnupg is installed",
+	}
+}
+
+// GPG1Finder looks for GnuPG1.x only
+func GPG1Finder() GPGFinder {
+	return &pathGPGFinder{
+		gpgNames:                 []string{"gpg", "gpg1"},
+		gpgvNames:                []string{"gpgv", "gpgv1"},
+		expectedVersionSubstring: "(GnuPG) 1.",
+		errorMessage:             "Couldn't find a suitable gpg executable. Make sure gnupg1 is available as either gpg(v) or gpg(v)1 in $PATH",
+	}
+}
+
+// GPG2Finder looks for GnuPG2.x only
+func GPG2Finder() GPGFinder {
+	return &pathGPGFinder{
+		gpgNames:                 []string{"gpg", "gpg2"},
+		gpgvNames:                []string{"gpgv", "gpgv2"},
+		expectedVersionSubstring: "(GnuPG) 2.",
+		errorMessage:             "Couldn't find a suitable gpg executable. Make sure gnupg2 is available as either gpg(v) or gpg(v)2 in $PATH",
+	}
+}
+
+func (pgf *pathGPGFinder) FindGPG() (gpg string, version GPGVersion, err error) {
+	for _, cmd := range pgf.gpgNames {
+		var result bool
+		result, version = cliVersionCheck(cmd, pgf.expectedVersionSubstring)
+		if result {
+			gpg = cmd
+			break
+		}
+	}
+
+	if gpg == "" {
+		err = errors.New(pgf.errorMessage)
+	}
+
+	return
+}
+
+func (pgf *pathGPGFinder) FindGPGV() (gpgv string, version GPGVersion, err error) {
+	for _, cmd := range pgf.gpgvNames {
+		var result bool
+		result, version = cliVersionCheck(cmd, pgf.expectedVersionSubstring)
+		if result {
+			gpgv = cmd
+			break
+		}
+	}
+
+	if gpgv == "" {
+		err = errors.New(pgf.errorMessage)
+	}
+
+	return
+}
+
+func (it *iteratingGPGFinder) FindGPG() (gpg string, version GPGVersion, err error) {
+	for _, finder := range it.finders {
+		gpg, version, err = finder.FindGPG()
+		if err == nil {
+			return
+		}
+	}
+
+	err = errors.New(it.errorMessage)
+
+	return
+}
+
+func (it *iteratingGPGFinder) FindGPGV() (gpg string, version GPGVersion, err error) {
+	for _, finder := range it.finders {
+		gpg, version, err = finder.FindGPGV()
+		if err == nil {
+			return
+		}
+	}
+
+	err = errors.New(it.errorMessage)
+
+	return
+}
+
+func cliVersionCheck(cmd string, marker string) (result bool, version GPGVersion) {
+	output, err := exec.Command(cmd, "--version").CombinedOutput()
+	if err != nil {
+		return
+	}
+
+	strOutput := string(output)
+	result = strings.Contains(strOutput, marker)
+
+	version = GPG21xPlus
+	matches := gpgVersionRegex.FindStringSubmatch(strOutput)
+	if matches != nil {
+		if matches[1] == "1" {
+			version = GPG1x
+		} else if matches[1] == "2" && matches[2] == "0" {
+			version = GPG20x
+		}
+	}
+
+	return
+}

pgp/gnupg_test.go

@@ -2,6 +2,7 @@ package pgp
 
 import (
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 
@@ -9,8 +10,7 @@ import (
 )
 
 type GnupgSuite struct {
-	verifier Verifier
+	bins string
-	bins     string
 }
 
 var _ = Suite(&GnupgSuite{})
@@ -26,7 +26,7 @@ func (s *GnupgSuite) TestGPG1(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpg1"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	signer := NewGpgSigner()
+	signer := NewGpgSigner(GPG1Finder())
 	c.Assert(signer.gpg, Equals, "gpg")
 }
 
@@ -36,7 +36,7 @@ func (s *GnupgSuite) TestGPG1Not2(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpg2-and-1"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	signer := NewGpgSigner()
+	signer := NewGpgSigner(GPG1Finder())
 	c.Assert(signer.gpg, Equals, "gpg1")
 }
 
@@ -46,7 +46,7 @@ func (s *GnupgSuite) TestGPGNothing(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpg2-only"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	c.Assert(func() { NewGpgSigner() }, PanicMatches, `Couldn't find a suitable gpg executable.+`)
+	c.Assert(func() { NewGpgSigner(GPG1Finder()) }, PanicMatches, `Couldn't find a suitable gpg executable.+`)
 }
 
 // If gpgv == gpgv1 = pick gpgv
@@ -55,7 +55,7 @@ func (s *GnupgSuite) TestGPGV1(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpgv1")+":"+filepath.Join(s.bins, "gpg1"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	verifier := NewGpgVerifier()
+	verifier := NewGpgVerifier(GPG1Finder())
 	c.Assert(verifier.gpgv, Equals, "gpgv")
 }
 
@@ -65,7 +65,7 @@ func (s *GnupgSuite) TestGPGV1Not2(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpgv2-and-1")+":"+filepath.Join(s.bins, "gpg1"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	verifier := NewGpgVerifier()
+	verifier := NewGpgVerifier(GPG1Finder())
 	c.Assert(verifier.gpgv, Equals, "gpgv1")
 }
 
@@ -75,5 +75,148 @@ func (s *GnupgSuite) TestGPGVNothing(c *C) {
 	os.Setenv("PATH", filepath.Join(s.bins, "gpgv2-only")+":"+filepath.Join(s.bins, "gpg1"))
 	defer func() { os.Setenv("PATH", origPath) }()
 
-	c.Assert(func() { NewGpgVerifier() }, PanicMatches, `Couldn't find a suitable gpgv executable.+`)
+	c.Assert(func() { NewGpgVerifier(GPG1Finder()) }, PanicMatches, `Couldn't find a suitable gpg executable.+`)
+}
+
+type Gnupg1VerifierSuite struct {
+	VerifierSuite
+}
+
+var _ = Suite(&Gnupg1VerifierSuite{})
+
+func (s *Gnupg1VerifierSuite) SetUpTest(c *C) {
+	finder := GPG1Finder()
+	_, _, err := finder.FindGPG()
+	if err != nil {
+		c.Skip(err.Error())
+	}
+
+	s.verifier = NewGpgVerifier(finder)
+	s.verifier.AddKeyring("./trusted.gpg")
+
+	c.Assert(s.verifier.InitKeyring(), IsNil)
+}
+
+type Gnupg1SignerSuite struct {
+	SignerSuite
+}
+
+var _ = Suite(&Gnupg1SignerSuite{})
+
+func (s *Gnupg1SignerSuite) SetUpTest(c *C) {
+	finder := GPG1Finder()
+	_, _, err := finder.FindGPG()
+	if err != nil {
+		c.Skip(err.Error())
+	}
+
+	s.keyringNoPassphrase = [2]string{"keyrings/aptly.pub", "keyrings/aptly.sec"}
+	s.keyringPassphrase = [2]string{"keyrings/aptly_passphrase.pub", "keyrings/aptly_passphrase.sec"}
+	s.passphraseKey = "F30E8CB9CDDE2AF8"
+	s.noPassphraseKey = "21DBB89C16DB3E6D"
+
+	s.signer = NewGpgSigner(finder)
+	s.signer.SetBatch(true)
+
+	s.verifier = &GoVerifier{}
+	s.verifier.AddKeyring("./keyrings/aptly.pub")
+	s.verifier.AddKeyring("./keyrings/aptly_passphrase.pub")
+
+	c.Assert(s.verifier.InitKeyring(), IsNil)
+
+	s.SignerSuite.SetUpTest(c)
+}
+
+type Gnupg2VerifierSuite struct {
+	VerifierSuite
+}
+
+var _ = Suite(&Gnupg2VerifierSuite{})
+
+func (s *Gnupg2VerifierSuite) SetUpTest(c *C) {
+	finder := GPG2Finder()
+	_, _, err := finder.FindGPG()
+	if err != nil {
+		c.Skip(err.Error())
+	}
+
+	s.verifier = NewGpgVerifier(finder)
+	s.verifier.AddKeyring("./trusted.gpg")
+
+	c.Assert(s.verifier.InitKeyring(), IsNil)
+}
+
+type Gnupg2SignerSuite struct {
+	SignerSuite
+}
+
+var _ = Suite(&Gnupg2SignerSuite{})
+
+func (s *Gnupg2SignerSuite) SetUpTest(c *C) {
+	finder := GPG2Finder()
+	gpg, ver, err := finder.FindGPG()
+	if err != nil {
+		c.Skip(err.Error())
+	}
+
+	// import private keys into gpg2, they're stored outside of keyring files
+	for _, item := range []struct {
+		suffix string
+		key    string
+	}{
+		{"", "751DF85C2B220D45"},
+		{"_passphrase", "6656CD181E92D2D5"},
+	} {
+		if _, err := exec.Command(gpg, "--list-secret-keys", item.key).CombinedOutput(); err == nil {
+			// key already exists
+			continue
+		}
+
+		args := []string{"--import", "--no-default-keyring"}
+
+		if item.suffix == "_passprhase" {
+			args = append(args, "--passphrase", "verysecret", "--no-tty", "--batch")
+			if ver == GPG21xPlus {
+				args = append(args, "--pinentry-mode", "loopback")
+			}
+		}
+		args = append(args, "keyrings/aptly2"+item.suffix+".sec.armor")
+
+		output, err := exec.Command(gpg, args...).CombinedOutput()
+		c.Log(string(output))
+		c.Check(err, IsNil)
+	}
+
+	// import public keys into gpg2
+	// we can't use pre-built keyrings as gpg 2.0.x and 2.1+ have different keyring formats
+	for _, suffix := range []string{"", "_passphrase"} {
+		output, err := exec.Command(gpg, "--no-default-keyring", "--keyring", "./keyrings/aptly2"+suffix+".gpg",
+			"--import", "keyrings/aptly2"+suffix+".pub.armor").CombinedOutput()
+		c.Log(string(output))
+		c.Check(err, IsNil)
+	}
+
+	s.keyringNoPassphrase = [2]string{"./keyrings/aptly2.gpg", ""}
+	s.keyringPassphrase = [2]string{"./keyrings/aptly2_passphrase.gpg", ""}
+	s.noPassphraseKey = "751DF85C2B220D45"
+	s.passphraseKey = "6656CD181E92D2D5"
+
+	s.signer = NewGpgSigner(finder)
+	s.signer.SetBatch(true)
+
+	s.verifier = &GoVerifier{}
+	s.verifier.AddKeyring("./keyrings/aptly2_trusted.pub")
+
+	c.Assert(s.verifier.InitKeyring(), IsNil)
+
+	s.skipDefaultKey = true
+
+	s.SignerSuite.SetUpTest(c)
+}
+
+func (s *Gnupg2SignerSuite) TearDownTest(c *C) {
+	s.SignerSuite.TearDownTest(c)
+
+	os.Remove("./keyrings/aptly2.gpg")
+	os.Remove("./keyrings/aptly2_passphrase.gpg")
 }

pgp/internal.go

@@ -367,7 +367,14 @@ func (g *GoVerifier) printLog(signers []signatureResult) {
 
 // VerifyDetachedSignature verifies combination of signature and cleartext using gpgv
 func (g *GoVerifier) VerifyDetachedSignature(signature, cleartext io.Reader, showKeyTip bool) error {
-	signers, missingKeys, err := checkArmoredDetachedSignature(g.trustedKeyring, cleartext, signature)
+	var signatureBuf bytes.Buffer
+
+	signers, missingKeys, err := checkArmoredDetachedSignature(g.trustedKeyring, cleartext, io.TeeReader(signature, &signatureBuf))
+
+	if err == io.EOF {
+		// most probably not armored signature
+		signers, missingKeys, err = checkDetachedSignature(g.trustedKeyring, cleartext, &signatureBuf)
+	}
 
 	g.printLog(signers)
 

pgp/internal_test.go

@@ -1,14 +1,11 @@
 package pgp
 
 import (
-	"io/ioutil"
-	"os"
-
 	. "gopkg.in/check.v1"
 )
 
 type GoVerifierSuite struct {
-	verifier Verifier
+	VerifierSuite
 }
 
 var _ = Suite(&GoVerifierSuite{})
@@ -20,77 +17,26 @@ func (s *GoVerifierSuite) SetUpTest(c *C) {
 	c.Assert(s.verifier.InitKeyring(), IsNil)
 }
 
-func (s *GoVerifierSuite) TestVerifyDetached(c *C) {
+type GoSignerSuite struct {
-	for _, test := range []struct {
+	SignerSuite
-		textName, signatureName string
-	}{
-		{"1.text", "1.signature"},
-		{"2.text", "2.signature"},
-		{"3.text", "3.signature"},
-	} {
-		cleartext, err := os.Open(test.textName)
-		c.Assert(err, IsNil)
-
-		signature, err := os.Open(test.signatureName)
-		c.Assert(err, IsNil)
-
-		err = s.verifier.VerifyDetachedSignature(signature, cleartext, false)
-		c.Assert(err, IsNil)
-
-		signature.Close()
-		cleartext.Close()
-	}
 }
 
-func (s *GoVerifierSuite) TestVerifyClearsigned(c *C) {
+var _ = Suite(&GoSignerSuite{})
-	for _, test := range []struct {
-		clearSignedName string
-	}{
-		{"1.clearsigned"},
-	} {
-		clearsigned, err := os.Open(test.clearSignedName)
-		c.Assert(err, IsNil)
 
-		keyInfo, err := s.verifier.VerifyClearsigned(clearsigned, false)
+func (s *GoSignerSuite) SetUpTest(c *C) {
-		c.Assert(err, IsNil)
+	s.keyringNoPassphrase = [2]string{"keyrings/aptly.pub", "keyrings/aptly.sec"}
-		c.Check(keyInfo.GoodKeys, DeepEquals, []Key{"8B48AD6246925553", "7638D0442B90D010"})
+	s.keyringPassphrase = [2]string{"keyrings/aptly_passphrase.pub", "keyrings/aptly_passphrase.sec"}
-		c.Check(keyInfo.MissingKeys, DeepEquals, []Key(nil))
+	s.passphraseKey = "F30E8CB9CDDE2AF8"
+	s.noPassphraseKey = "21DBB89C16DB3E6D"
 
-		clearsigned.Close()
+	s.signer = &GoSigner{}
-	}
+	s.signer.SetBatch(true)
-}
+
-
+	s.verifier = &GoVerifier{}
-func (s *GoVerifierSuite) TestExtractClearsigned(c *C) {
+	s.verifier.AddKeyring("./keyrings/aptly.pub")
-	for _, test := range []struct {
+	s.verifier.AddKeyring("./keyrings/aptly_passphrase.pub")
-		clearSignedName, clearTextName string
+
-	}{
+	c.Assert(s.verifier.InitKeyring(), IsNil)
-		{"1.clearsigned", "1.cleartext"},
+
-	} {
+	s.SignerSuite.SetUpTest(c)
-		clearsigned, err := os.Open(test.clearSignedName)
-		c.Assert(err, IsNil)
-
-		cleartext, err := os.Open(test.clearTextName)
-		c.Assert(err, IsNil)
-
-		is, err := s.verifier.IsClearSigned(clearsigned)
-		c.Assert(err, IsNil)
-		c.Check(is, Equals, true)
-
-		clearsigned.Seek(0, 0)
-
-		extractedF, err := s.verifier.ExtractClearsigned(clearsigned)
-		c.Assert(err, IsNil)
-
-		expected, err := ioutil.ReadAll(cleartext)
-		c.Assert(err, IsNil)
-
-		extracted, err := ioutil.ReadAll(extractedF)
-		c.Assert(err, IsNil)
-
-		c.Check(expected, DeepEquals, extracted)
-
-		extractedF.Close()
-		clearsigned.Close()
-		cleartext.Close()
-	}
 }

pgp/keyrings/aptly2.pub.armor

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBFuAfvMBCACuElVMZctaYEMSIoYLAefC2ygH/MUWA41h/MT/kMakexK7B2N/
+noPUhA0Z7xWiqxnvJqaOGuT2Bp2SSOuw8hXD870VMNNAVuvg63Zvj5DGhWAjR8Sm
+zHaZ09+hkD+MB7+WJnyDJb0deGpJ7cFaUgS4fz1BlTWHpJX8wMBi3iqA2tulcoNn
+L/pUomusNGiD+4TuWkEeYb3/ygXvfocuE0Ji7UFrijU4Dcrh7T7L7qzHDMy8hyEr
+t3oZlFDlwRkr+1LrT1QBnndaddPRt1h3Av59WpasTUC8m/It0NvLpq9mqij3TNTx
+OyZNJLqHrUsz1/cg3boiT4puY8gm2jpQpNLXABEBAAG0HkFwdGx5IFRlc3RlciA8
+dGVzdEBhcHRseS5pbmZvPokBTgQTAQgAOBYhBOivfuFBYsLWPPwa1XUd+FwrIg1F
+BQJbgH7zAhsDBQsJCAcCBhUICQoLAgQWAgMBAh4BAheAAAoJEHUd+FwrIg1FWRwH
+/RmXNwgh6DEj7wN7hILL15iXOrOIjEJ2GH0cWvPdyzc/qbL552QovcaK3yb/r3V+
++1wDJ2CuBtc/CoVFq1CN/i92wIDl+Cuozny5qcd8O6EjgdmLgeANRwqyMjiPiDz9
+cuabD3JPRHIqEv26PQ+qkmad42E5mipHmbA+iOE9OEWSvhDudAlYzNXECUWlNQ9+
+gWnLB6hONz5jnRDZHpcKeBcQ2aJ7r5L6qDzBIybAu3jiZfl5KlT6hArXi5vDi8DK
+B5is80nWPTAEb2+CfBiY80mLScNe5jG/sgOOrTqWL681RfjRtTnRe7DFKIm8guqp
+tbYrv4OzkFHJ/JbWAKsBruS5AQ0EW4B+8wEIANWaf4BWY2or9oyu001EmIdFiwu2
+cxGA2y8bZiqmerk+2BXDEZN4OaLu1a9RWpwo1Mc+KuXpeJNv60SG0zRFBLVrvPyg
+irhaue1p+SSuisxMdTOZrciYjWriTU4WKw+NOdiGHr5LJegE9hvW66ZYJHtYgkfB
+mBuIQQ90h6qnXKGtV4FK8Fo+hr04Wh7gDGZxTRFNo3MO0a18Y87uiU5j8i/VxyfI
+DSA2Uh92kPbItuEKtl23PhCSecZa4YkkWMILS7frMEbM9wDK/JFqPVPSUwQhV5jn
+wwq9hwQrUimrhZjJn1EImK2QVYeJ1CVxc3K7bdlxPd8fi6zfTQ8AfpALDWkAEQEA
+AYkBNgQYAQgAIBYhBOivfuFBYsLWPPwa1XUd+FwrIg1FBQJbgH7zAhsMAAoJEHUd
++FwrIg1FzpEH/2xi/DCaYRgbC4RrICebeC8FBTwI2RyBuOQJr5CIPrpWaWV4+5Ds
+sIgPxU9E3QgNRjP9pzAzH2Z8WwJtRY0oYNWLFruNeg9Xl1Tf9pCK/0Csamyf/h3F
+6NKfDTwNBWTsD5ttNyRx2nfDPaU4j2BZqU3kOzdwiXnmtvtxEoH059EMgQFLv91W
+U7NydHYd8xcWlHIZx1uFB8HKRWB+AMXebkdLVXlUtJfZfxZr5Jb5eR77ojfsduYI
+YmbV4jkDxuidSkYogYyXMO2jY2PAhx29iaZrDsNdsCsg8OmwEjCPEGWdp7tFA1QI
+JrGSOnwXujoUwuqh53+n3bcVuuuKyPBW9b4=
+=Tu5z
+-----END PGP PUBLIC KEY BLOCK-----

pgp/keyrings/aptly2.sec.armor

@@ -0,0 +1,57 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lQOYBFuAfvMBCACuElVMZctaYEMSIoYLAefC2ygH/MUWA41h/MT/kMakexK7B2N/
+noPUhA0Z7xWiqxnvJqaOGuT2Bp2SSOuw8hXD870VMNNAVuvg63Zvj5DGhWAjR8Sm
+zHaZ09+hkD+MB7+WJnyDJb0deGpJ7cFaUgS4fz1BlTWHpJX8wMBi3iqA2tulcoNn
+L/pUomusNGiD+4TuWkEeYb3/ygXvfocuE0Ji7UFrijU4Dcrh7T7L7qzHDMy8hyEr
+t3oZlFDlwRkr+1LrT1QBnndaddPRt1h3Av59WpasTUC8m/It0NvLpq9mqij3TNTx
+OyZNJLqHrUsz1/cg3boiT4puY8gm2jpQpNLXABEBAAEAB/0Uwor5uYovFRvqpcNm
+vKtvScPUcAjxPys1bHfyIsoOA7+QHql3JuiCB92oIiNqaw2EwA9NE1gLH27ba2bw
+i26dGAyM4m3PVo57HJnbZDvX8UAt9Pk3C1t5rRMWqaVqheILfjuLSIztXtcOShSt
+OOrnNgWQNMNVkgNxSWuoXuaix0S0JrwuMdbrzYeiXsiBKOR9Z72vrj27aj7siQBP
+Otn7iUlWNKH5mnEGd4O36oVpiLZ2QjG0Xr5ZLxCheKa2aMP0udhSjjg/t2lTuNrG
+rWjGDtYF7QGXVcnSz0VI9a/GKA+9wmfTxutq4mkZywoGo4YRlXVeOmKPxh5hJJaJ
+hw6BBADCRdQDGbvSnyHAM95ljseE8Pf6E7O2d06CjP/QjFVB/xQ+8FLeM//G2JK8
+3FUa5mx5vXBrfAaf+odXhlMgLZa3seiBoywpQR11L5TlXo7pGht6GwjEbcCqqBIF
+TM4XAYN9CMHe2kvfEoIWxdt8g+G+htcTV4ESh+50Llczf0RqLwQA5WFXOBVx5C7v
+6yVTTBCrj6LYnL5NKhD/wpK116O8IKwPWgPMbViLcL9xwaCGh4aM7zKvvAWe8ffU
+10qv/1ApkqjVwXSBJhOXe+Cv/j86MZKvpjzLtQaOustooMXCCXb5aOyjaKedSlB0
+ZHiUGyTn6t/fHq393otAMnSdgHW+/9kEAMGW1tdYvImDYWX54FcGeeHsZxfN9OZE
+NGpksi/dDP+bh0ykIRryWc673ATwZ5wdjG8BnV0Gn4tEMe2T4RCq+DhB3dSPyRn0
+FFueWUrhH1xIU2ntbyCdzOuPgnVj8RKUfbi85ANnXfbmRgZtnaxRDxhn7ac4zby0
+POvyH/yyA+UyRXm0HkFwdGx5IFRlc3RlciA8dGVzdEBhcHRseS5pbmZvPokBTgQT
+AQgAOBYhBOivfuFBYsLWPPwa1XUd+FwrIg1FBQJbgH7zAhsDBQsJCAcCBhUICQoL
+AgQWAgMBAh4BAheAAAoJEHUd+FwrIg1FWRwH/RmXNwgh6DEj7wN7hILL15iXOrOI
+jEJ2GH0cWvPdyzc/qbL552QovcaK3yb/r3V++1wDJ2CuBtc/CoVFq1CN/i92wIDl
++Cuozny5qcd8O6EjgdmLgeANRwqyMjiPiDz9cuabD3JPRHIqEv26PQ+qkmad42E5
+mipHmbA+iOE9OEWSvhDudAlYzNXECUWlNQ9+gWnLB6hONz5jnRDZHpcKeBcQ2aJ7
+r5L6qDzBIybAu3jiZfl5KlT6hArXi5vDi8DKB5is80nWPTAEb2+CfBiY80mLScNe
+5jG/sgOOrTqWL681RfjRtTnRe7DFKIm8guqptbYrv4OzkFHJ/JbWAKsBruSdA5cE
+W4B+8wEIANWaf4BWY2or9oyu001EmIdFiwu2cxGA2y8bZiqmerk+2BXDEZN4OaLu
+1a9RWpwo1Mc+KuXpeJNv60SG0zRFBLVrvPygirhaue1p+SSuisxMdTOZrciYjWri
+TU4WKw+NOdiGHr5LJegE9hvW66ZYJHtYgkfBmBuIQQ90h6qnXKGtV4FK8Fo+hr04
+Wh7gDGZxTRFNo3MO0a18Y87uiU5j8i/VxyfIDSA2Uh92kPbItuEKtl23PhCSecZa
+4YkkWMILS7frMEbM9wDK/JFqPVPSUwQhV5jnwwq9hwQrUimrhZjJn1EImK2QVYeJ
+1CVxc3K7bdlxPd8fi6zfTQ8AfpALDWkAEQEAAQAH+K33puBfe5h9NdBekrnbpF6H
+xTdE4XLf/6PeLNePv2QgSt1ugmIZCNgqrN6c469LkgC0ITwfapSqEnM9W8a2b59S
+oBkgp9p+Ce/S35eAkIrTuqDMCT3XAVaL+Wofo/KGkxZGJcPWcIkHgWorIMHaB9xt
+ua23fqrtzg9IWTYkGM2TYz/Kf+VbrPTuXoRxbf0skOyCHDNaP+Xr05/e/CxCM4//
+IigMa5qHSk+eO+YP/dAvSfWmERax2juD//jC4cazoLv/WzeZtsyM+QQKPqYAH6yk
+Hwcwb+bCKwieLOHX215DS2v+ZWvS9KQKPd/LJclxlEcNBnBaaGGT4O+9NhP16wQA
+5qnPeRRZI2CR4srFQJSPlI86ONjKX4TP1tpNUq3v+cLRzuX3rw7iYXr4hiJnWF7o
+NPUA6U4kMVSCyvbdQ/+CD3nuua5UPQR7ghJKzVfj0Hx2CxaD4noCaB7ptmJj+Pn3
+ZogSGjQ7nKJntV6BGRYnnHXmucjZsepyfXGBPko9vK8EAO0Q+GXaov+Fpq4HmLZq
+nh+GSCu6PIa5WMhy2iwi6pvx/ZcSUc7XEcSjYIT6FCbLlmis7sdE31CDzzVISbdm
+WNq1i2SHVdFwA6JRrYtg7PKpG+UqOFUYx1BDcnbxndEeXf48oznSE8yVVJrkXPzy
+3349UaopdBUFW+qcH+nvpM1nA/4zb2K5aOWWeSQOEvaX7v0TFcAHdsE2jZ4e6Gvg
+ixNrvLqfqlX90IUUquyzus486iC9CELAm2DK0UdV+SVxSxw39HulnUZBPQnvVbXu
+aE0aRO6Jidl8nWm39dfzGxa+7HJxdTA6LIlpp/Ol7FaHmSSYxynjtjmhgLuGG3Z8
+U59cO07XiQE2BBgBCAAgFiEE6K9+4UFiwtY8/BrVdR34XCsiDUUFAluAfvMCGwwA
+CgkQdR34XCsiDUXOkQf/bGL8MJphGBsLhGsgJ5t4LwUFPAjZHIG45AmvkIg+ulZp
+ZXj7kOywiA/FT0TdCA1GM/2nMDMfZnxbAm1FjShg1YsWu416D1eXVN/2kIr/QKxq
+bJ/+HcXo0p8NPA0FZOwPm203JHHad8M9pTiPYFmpTeQ7N3CJeea2+3ESgfTn0QyB
+AUu/3VZTs3J0dh3zFxaUchnHW4UHwcpFYH4Axd5uR0tVeVS0l9l/Fmvklvl5Hvui
+N+x25ghiZtXiOQPG6J1KRiiBjJcw7aNjY8CHHb2JpmsOw12wKyDw6bASMI8QZZ2n
+u0UDVAgmsZI6fBe6OhTC6qHnf6fdtxW664rI8Fb1vg==
+=vqFY
+-----END PGP PRIVATE KEY BLOCK-----

pgp/keyrings/aptly2_passphrase.pub.armor

@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBFuAgf0BCADhVDnKyR5G6RzgFb7QseLqOzjvGotmbYYTuYinwLC+GGgOBD54
+DTtzBk6PmU1/QW7x0yvffyeQWUXD+/zIuBLrkG2QXv9qRdADH6rCsVChtVsTRwHT
+ZgJJXgfcjsZ2UUDQ43Jkb4dMaAIJTS0dASXgyLIRaN/1h4SsdKZUZY16lMH9kSxb
+XI86qujCdC9WS7CWu8sLX7qKqmmAt8fTGTnTgyQCnIYtS1/je0ZfX4Z3ovzYyajJ
+LA/jtCD6L+3qj9Y7desj1AvNKRCGYB8nzayYYVqozNFvlubDePinFqJThrICviNS
+cNZuNmZjToPmSMF/B6zz1CkrR6Q0CIM5zkuxABEBAAG0HkFwdGx5IFRlc3RlciA8
+dGVzdEBhcHRseS5pbmZvPokBTgQTAQgAOBYhBAvV0vLk0JvPtsmaFGZWzRgektLV
+BQJbgIH9AhsDBQsJCAcCBhUICQoLAgQWAgMBAh4BAheAAAoJEGZWzRgektLVAHoH
+/jwdhMCDmgBR3vhS/i7ym5HFUx7o1cCv0cSoGQwgnRGLrTG4Ua4ds+FxAKjNAspC
+hgxGGfTCmD1cYB4lx85I0XcpfT4ZA0ZCQ1I9m5/AJ9WWYoy2FFZNxZ7fCd2d+sm1
+ydyJLCTRgJd0D2MD58vxF2+IWAkZTOridyyZP1qxfzcnSACTjbcrQFc1Bp7G1xJR
+T135mcxtBSD2JeXbvq/UkeOE+LCIF5EIHUlwOdanyyHTyO86R2rU7qO0bylNyLGi
+0SE3Y+j+qhg6Ns4I2SPo0JDBDrSVfPKcob1DYFF0K1yH6Cj3aGVUthsahNrP3K9S
+NOVRC1JUg5an0eWcayIPpte5AQ0EW4CB/QEIAMrLVpvc2Syuhtov6mBTuIB9SUHy
+9tyqK6WhyDAB7iQPC5Xrb4GXHM4z+2Vt7Pgabr630B6ySsmrKNfDB1EcUcY4aJ5Q
+LBIttR25CB8PkmvllycjKp49hLXOruwx6t23/J3REyapQXLKpXDhzmMTmoGpAPcu
+YTVMiiOFi0MNBI5ok9iQoG9Dbgf7F0Ie1MiLJzqfaf2dlRuh2JBayIMJ9VrtlThB
+5v2CsV73Mj7EgE7oetZgH2I9Tl/2vv70VYnoXn/YL2xgfghWWlTvDTvg9s7sRnDI
+4oor7SHaRZIWxqSuvVOchnlBjLADn902bFIPEanqBiAnPAHPwPOMKnBoR78AEQEA
+AYkBNgQYAQgAIBYhBAvV0vLk0JvPtsmaFGZWzRgektLVBQJbgIH9AhsMAAoJEGZW
+zRgektLVetQH/1QiJqu6JmEbJ4Z8mmwr25iqeC/9G1Llo6q4VAoLFygFU1EVW+wX
+tPwz5RZpqoZ5K4NVFJcgLH1ExqgKW9MflPK+6ltYqH3Kgg73I5gnP9SX12hJ3oMZ
+X15uGm8pN+na4fcu9rwvk87EBfXXmtP9oqdCPZ49DtkngyahTXFK/h5CbcgCx1Fw
+rm1QXyH/sB8O3f6GOVly40u7CV0hzWAB98KLgEP093OI3AJjmINxErCt6cigxQzO
+jgbiTMVgLKHidnpeOcKNXIlmIBFpBM4oaKFZ0qt5jiKz/QtmX7Uj+rTfCjaC1G7f
+fGE72fE3nL5TcbwRyD7g9zRV3ddU5uvCnlg=
+=075D
+-----END PGP PUBLIC KEY BLOCK-----

pgp/keyrings/aptly2_passphrase.sec.armor

@@ -0,0 +1,59 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lQPGBFuAgf0BCADhVDnKyR5G6RzgFb7QseLqOzjvGotmbYYTuYinwLC+GGgOBD54
+DTtzBk6PmU1/QW7x0yvffyeQWUXD+/zIuBLrkG2QXv9qRdADH6rCsVChtVsTRwHT
+ZgJJXgfcjsZ2UUDQ43Jkb4dMaAIJTS0dASXgyLIRaN/1h4SsdKZUZY16lMH9kSxb
+XI86qujCdC9WS7CWu8sLX7qKqmmAt8fTGTnTgyQCnIYtS1/je0ZfX4Z3ovzYyajJ
+LA/jtCD6L+3qj9Y7desj1AvNKRCGYB8nzayYYVqozNFvlubDePinFqJThrICviNS
+cNZuNmZjToPmSMF/B6zz1CkrR6Q0CIM5zkuxABEBAAH+BwMCCNSe9dK/Hnvnv40X
+biknM8EGfz7o+DqsB0TOwAZnYCtp6rAWymEaZoukf2gIiI5P2swfzvEwfmbTopDg
+9wx23IEtinPy06lqyVlrumGsPrJ3DX86p0O0N4VeSu/A2pxUafuLFhrAmY8uX+mH
+Dd1gycTm8LifSPyfP6/ce82ktNBrC4ICk82j4r949oXva4vPombv7b8YEAuxbIus
+cDrA6ot4ez1y55PIM+9TphEDRVeWjh4baMjO2Q5uDk+qGzOfzdUai4uw1VBT9hWs
+xG/7wWMoFWhxlYX8GfCZBMKP7B41+1yFQ+D9EHHyxJeXlo51b7XvuTPiGZ1vgm8+
+Xj9/xuTwpEv42s1W/ItJukEEV6EeQUkAEBZtX6SInsZmDVKFUPFl/I/IlK99eXdH
+uwv4G/6OwY3wZemv78ugNf2jEoHR+gl6JsPw9XiSj4jSOAbRTEhtQa6TOOFmBv/9
+P3i4gWCbiSDInjnA/MKplvIsNgjwe+YksWZuROt15QFHPsXJ+J8BZ2AcIL1ZbHB7
+m1qikSK9x+4zbGl9w8AaaYrVtKWT0ds1+1Sauw7mIcE6VnAd5kscGqHzJuogWH6t
+VAs+E3fNirDYmb36RPVUd+fbBRo81QRlVbC8Ef3bKbQzFl9geaiUZGNzHqCfc+La
+Hc7ruk3f8/h0bWHqm8Rb3kiDl9zVSVg0qWGCIwgzKowgfqAvyvapKXYPsMLLS9W/
+2RrMO9DsUZDznHkUmr6njaBF7evqmy5KIBpyeIf/uIOzP8qI2KbVlucZZfl+teqy
+lhN9uECbZkZZofZVcQha2piuOWgDGCpuz15UuLbvsPhOKtKmnt2ITnTOAkI47t2h
+zOpxommMx8s/MGP8dEwcrAB6Ao/dORN6lVr+kPAOKV/KSI5iQ9z8tWjdL6hgsocQ
+Ds7DyGhp8D2VtB5BcHRseSBUZXN0ZXIgPHRlc3RAYXB0bHkuaW5mbz6JAU4EEwEI
+ADgWIQQL1dLy5NCbz7bJmhRmVs0YHpLS1QUCW4CB/QIbAwULCQgHAgYVCAkKCwIE
+FgIDAQIeAQIXgAAKCRBmVs0YHpLS1QB6B/48HYTAg5oAUd74Uv4u8puRxVMe6NXA
+r9HEqBkMIJ0Ri60xuFGuHbPhcQCozQLKQoYMRhn0wpg9XGAeJcfOSNF3KX0+GQNG
+QkNSPZufwCfVlmKMthRWTcWe3wndnfrJtcnciSwk0YCXdA9jA+fL8RdviFgJGUzq
+4ncsmT9asX83J0gAk423K0BXNQaextcSUU9d+ZnMbQUg9iXl276v1JHjhPiwiBeR
+CB1JcDnWp8sh08jvOkdq1O6jtG8pTcixotEhN2Po/qoYOjbOCNkj6NCQwQ60lXzy
+nKG9Q2BRdCtch+go92hlVLYbGoTaz9yvUjTlUQtSVIOWp9HlnGsiD6bXnQPGBFuA
+gf0BCADKy1ab3NksrobaL+pgU7iAfUlB8vbcqiulocgwAe4kDwuV62+BlxzOM/tl
+bez4Gm6+t9AeskrJqyjXwwdRHFHGOGieUCwSLbUduQgfD5Jr5ZcnIyqePYS1zq7s
+Merdt/yd0RMmqUFyyqVw4c5jE5qBqQD3LmE1TIojhYtDDQSOaJPYkKBvQ24H+xdC
+HtTIiyc6n2n9nZUbodiQWsiDCfVa7ZU4Qeb9grFe9zI+xIBO6HrWYB9iPU5f9r7+
+9FWJ6F5/2C9sYH4IVlpU7w074PbO7EZwyOKKK+0h2kWSFsakrr1TnIZ5QYywA5/d
+NmxSDxGp6gYgJzwBz8DzjCpwaEe/ABEBAAH+BwMCcKOiS51jPV/nio4h+yCyOsXu
+Gz+LIgKHZ7neaiRGyz+8Q/+7b9ydbK29rYTV0TO0lVoyeS8hjaLmwRr+Um0v4AKi
+wzB+ca+cBKFbYEx4XyN0kywpQAZ8vcWpe8yyPTdudYh8q3v+egEEkdpgRQphYRVu
+mxGLCu3ay6wfeuUI2xTBGKDRaiNvPT+GaH+5QimiFg8slmAKRDO3DsQ2QXxV3X10
+/cOOao0kJMXcwmyiLC+ziK50VxPltYZetZ6nuERDcs5B2C89dCQ8zn+pxGTbC5fp
+tksh2giHzls7Jn5Kbd4A2HUR9M0wFCJBVKEqQF2D2IBjCA7tIvq4YG5GKHNgWAgC
+qfulC7XNiITYXyd4ac7Xng53fbxqD9vyDo96pzyaiSU4461PuKahB1jY7fivrKNa
+G5f37xIDhj8+ZH28FR4n+MxS9H93Ppde4sb8o786d9EmxJJZIZT8ncJEqukL2VjV
+SkIujadVirTT8OWrh9avq3CbQK+CJX179JfIeFiPsmEN8RVERKxXUkrmwD8c8TAa
+/ENk8i2bRCke5Tcw4E/fe9mkMOmYQVpZwbWnr4tAwfb310VgY8yWy9j7C/Iy4eTW
+NWO1qCenOLygzbOfQnzeo0tKiHCJpsTgY9UqnxJ/U1DpBbQS9AU22U4acAscnJuZ
+T6vx0TK/u0BVMN6TLQiRhjYzqTo5vyhVatKlhG5IuCiejAM4zT9h64C5ewk7B1l8
+h6k2gXy7Ba03Lz9t+UGuLpYDFVtoJmjmqzzgnYjPuASKVS4CPtzI/7p1L+d0sHp3
+5nOTgqRyIiGYk9RqQK7F+Bre4Vn6cydAcdiAwTt0910JuJchm3+d7GMiy3ntUIce
+LG4hL8T62/n5JHb0FXnSUIS07JyJseSuu++qco0QEwwmiIeXcA76PVy3ty166r01
+CFlZiQE2BBgBCAAgFiEEC9XS8uTQm8+2yZoUZlbNGB6S0tUFAluAgf0CGwwACgkQ
+ZlbNGB6S0tV61Af/VCImq7omYRsnhnyabCvbmKp4L/0bUuWjqrhUCgsXKAVTURVb
+7Be0/DPlFmmqhnkrg1UUlyAsfUTGqApb0x+U8r7qW1iofcqCDvcjmCc/1JfXaEne
+gxlfXm4abyk36drh9y72vC+TzsQF9dea0/2ip0I9nj0O2SeDJqFNcUr+HkJtyALH
+UXCubVBfIf+wHw7d/oY5WXLjS7sJXSHNYAH3wouAQ/T3c4jcAmOYg3ESsK3pyKDF
+DM6OBuJMxWAsoeJ2el45wo1ciWYgEWkEzihooVnSq3mOIrP9C2ZftSP6tN8KNoLU
+bt98YTvZ8TecvlNxvBHIPuD3NFXd11Tm68KeWA==
+=YDL1
+-----END PGP PRIVATE KEY BLOCK-----

pgp/sign_test.go

@@ -0,0 +1,190 @@
+package pgp
+
+import (
+	"crypto/rand"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	. "gopkg.in/check.v1"
+)
+
+// Common set of tests shared by internal & external GnuPG implementations
+type SignerSuite struct {
+	signer   Signer
+	verifier Verifier
+
+	clearF    *os.File
+	signedF   *os.File
+	cleartext []byte
+
+	passwordFile string
+
+	skipDefaultKey bool
+
+	keyringNoPassphrase [2]string
+	keyringPassphrase   [2]string
+
+	noPassphraseKey Key
+	passphraseKey   Key
+}
+
+func (s *SignerSuite) SetUpTest(c *C) {
+	tempDir := c.MkDir()
+
+	var err error
+	s.clearF, err = os.Create(path.Join(tempDir, "cleartext"))
+	c.Assert(err, IsNil)
+
+	s.cleartext = make([]byte, 0, 1024)
+	_, err = rand.Read(s.cleartext)
+	c.Assert(err, IsNil)
+
+	_, err = s.clearF.Write(s.cleartext)
+	c.Assert(err, IsNil)
+
+	_, err = s.clearF.Seek(0, io.SeekStart)
+	c.Assert(err, IsNil)
+
+	s.signedF, err = os.Create(path.Join(tempDir, "signed"))
+	c.Assert(err, IsNil)
+
+	s.passwordFile = path.Join(tempDir, "password")
+	f, err := os.OpenFile(s.passwordFile, os.O_CREATE|os.O_WRONLY, 0600)
+	c.Assert(err, IsNil)
+
+	_, err = f.Write([]byte("verysecret"))
+	c.Assert(err, IsNil)
+
+	f.Close()
+
+	s.signer.SetBatch(true)
+}
+
+func (s *SignerSuite) TearDownTest(c *C) {
+	s.clearF.Close()
+	s.signedF.Close()
+}
+
+func (s *SignerSuite) testSignDetached(c *C) {
+	c.Assert(s.signer.Init(), IsNil)
+
+	err := s.signer.DetachedSign(s.clearF.Name(), s.signedF.Name())
+	c.Assert(err, IsNil)
+
+	err = s.verifier.VerifyDetachedSignature(s.signedF, s.clearF, false)
+	c.Assert(err, IsNil)
+}
+
+func (s *SignerSuite) TestSignDetachedNoPassphrase(c *C) {
+	s.signer.SetKey(string(s.noPassphraseKey))
+	s.signer.SetKeyRing(s.keyringNoPassphrase[0], s.keyringNoPassphrase[1])
+
+	s.testSignDetached(c)
+}
+
+func (s *SignerSuite) TestSignDetachedNoPassphraseDefaultKey(c *C) {
+	if s.skipDefaultKey {
+		c.Skip("test for default key skipped")
+	}
+
+	s.signer.SetKeyRing(s.keyringNoPassphrase[0], s.keyringNoPassphrase[1])
+
+	s.testSignDetached(c)
+}
+
+func (s *SignerSuite) TestSignDetachedPassphrase(c *C) {
+	s.signer.SetKey(string(s.passphraseKey))
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("verysecret", "")
+
+	s.testSignDetached(c)
+}
+
+func (s *SignerSuite) TestSignDetachedPassphraseDefaultKey(c *C) {
+	if s.skipDefaultKey {
+		c.Skip("test for default key skipped")
+	}
+
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("verysecret", "")
+
+	s.testSignDetached(c)
+}
+
+func (s *SignerSuite) TestSignDetachedPassphraseFile(c *C) {
+	s.signer.SetKey(string(s.passphraseKey))
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("", s.passwordFile)
+
+	s.testSignDetached(c)
+}
+
+func (s *SignerSuite) testClearSign(c *C, expectedKey Key) {
+	c.Assert(s.signer.Init(), IsNil)
+
+	err := s.signer.ClearSign(s.clearF.Name(), s.signedF.Name())
+	c.Assert(err, IsNil)
+
+	keyInfo, err := s.verifier.VerifyClearsigned(s.signedF, false)
+	c.Assert(err, IsNil)
+
+	c.Assert(keyInfo.GoodKeys, DeepEquals, []Key{expectedKey})
+	c.Assert(keyInfo.MissingKeys, DeepEquals, []Key(nil))
+
+	_, err = s.signedF.Seek(0, io.SeekStart)
+	c.Assert(err, IsNil)
+	extractedF, err := s.verifier.ExtractClearsigned(s.signedF)
+	c.Assert(err, IsNil)
+	defer extractedF.Close()
+
+	extracted, err := ioutil.ReadAll(extractedF)
+	c.Assert(err, IsNil)
+
+	c.Assert(extracted, DeepEquals, s.cleartext)
+}
+
+func (s *SignerSuite) TestClearSignNoPassphrase(c *C) {
+	s.signer.SetKey(string(s.noPassphraseKey))
+	s.signer.SetKeyRing(s.keyringNoPassphrase[0], s.keyringNoPassphrase[1])
+
+	s.testClearSign(c, s.noPassphraseKey)
+}
+
+func (s *SignerSuite) TestClearSignNoPassphraseDefaultKey(c *C) {
+	if s.skipDefaultKey {
+		c.Skip("test for default key skipped")
+	}
+
+	s.signer.SetKeyRing(s.keyringNoPassphrase[0], s.keyringNoPassphrase[1])
+
+	s.testClearSign(c, s.noPassphraseKey)
+}
+
+func (s *SignerSuite) TestClearSignPassphrase(c *C) {
+	s.signer.SetKey(string(s.passphraseKey))
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("verysecret", "")
+
+	s.testClearSign(c, s.passphraseKey)
+}
+
+func (s *SignerSuite) TestClearSignPassphraseDefaultKey(c *C) {
+	if s.skipDefaultKey {
+		c.Skip("test for default key skipped")
+	}
+
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("verysecret", "")
+
+	s.testClearSign(c, s.passphraseKey)
+}
+
+func (s *SignerSuite) TestClearSignPassphraseFile(c *C) {
+	s.signer.SetKey(string(s.passphraseKey))
+	s.signer.SetKeyRing(s.keyringPassphrase[0], s.keyringPassphrase[1])
+	s.signer.SetPassphrase("", s.passwordFile)
+
+	s.testClearSign(c, s.passphraseKey)
+}

pgp/verify_test.go

@@ -0,0 +1,94 @@
+package pgp
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+
+	. "gopkg.in/check.v1"
+)
+
+// Common set of tests shared by internal & external GnuPG implementations
+type VerifierSuite struct {
+	verifier Verifier
+}
+
+func (s *VerifierSuite) TestVerifyDetached(c *C) {
+	for _, test := range []struct {
+		textName, signatureName string
+	}{
+		{"1.text", "1.signature"},
+		{"2.text", "2.signature"},
+		{"3.text", "3.signature"},
+		{"4.text", "4.signature"},
+	} {
+		cleartext, err := os.Open(test.textName)
+		c.Assert(err, IsNil)
+
+		signature, err := os.Open(test.signatureName)
+		c.Assert(err, IsNil)
+
+		err = s.verifier.VerifyDetachedSignature(signature, cleartext, false)
+		c.Assert(err, IsNil)
+
+		signature.Close()
+		cleartext.Close()
+	}
+}
+
+func (s *VerifierSuite) TestVerifyClearsigned(c *C) {
+	for _, test := range []struct {
+		clearSignedName string
+	}{
+		{"1.clearsigned"},
+	} {
+		clearsigned, err := os.Open(test.clearSignedName)
+		c.Assert(err, IsNil)
+
+		keyInfo, err := s.verifier.VerifyClearsigned(clearsigned, false)
+		c.Assert(err, IsNil)
+		c.Check(keyInfo.GoodKeys, DeepEquals, []Key{"8B48AD6246925553", "7638D0442B90D010"})
+		c.Check(keyInfo.MissingKeys, DeepEquals, []Key(nil))
+
+		clearsigned.Close()
+	}
+}
+
+func (s *VerifierSuite) TestExtractClearsigned(c *C) {
+	for _, test := range []struct {
+		clearSignedName, clearTextName string
+	}{
+		{"1.clearsigned", "1.cleartext"},
+	} {
+		clearsigned, err := os.Open(test.clearSignedName)
+		c.Assert(err, IsNil)
+
+		cleartext, err := os.Open(test.clearTextName)
+		c.Assert(err, IsNil)
+
+		is, err := s.verifier.IsClearSigned(clearsigned)
+		c.Assert(err, IsNil)
+		c.Check(is, Equals, true)
+
+		clearsigned.Seek(0, 0)
+
+		extractedF, err := s.verifier.ExtractClearsigned(clearsigned)
+		c.Assert(err, IsNil)
+
+		expected, err := ioutil.ReadAll(cleartext)
+		c.Assert(err, IsNil)
+
+		extracted, err := ioutil.ReadAll(extractedF)
+		c.Assert(err, IsNil)
+
+		// normalize newlines
+		extracted = bytes.TrimRight(bytes.Replace(extracted, []byte("\r\n"), []byte("\n"), -1), "\n")
+		expected = bytes.Replace(expected, []byte("\r\n"), []byte("\n"), -1)
+
+		c.Check(extracted, DeepEquals, expected)
+
+		extractedF.Close()
+		clearsigned.Close()
+		cleartext.Close()
+	}
+}

query/lex.go

@@ -119,14 +119,6 @@ func (l *lexer) backup() {
 	l.pos -= l.width
 }
 
-// peek returns but does not consume
-// the next rune in the input.
-func (l *lexer) peek() rune {
-	r := l.next()
-	l.backup()
-	return r
-}
-
 func (l *lexer) Current() item {
 	if l.last.typ == 0 {
 		l.last = <-l.items

s3/public.go

@@ -17,7 +17,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/pkg/errors"
-	"github.com/smira/go-aws-auth"
+	awsauth "github.com/smira/go-aws-auth"
 )
 
 const errCodeNotFound = "NotFound"
@@ -190,6 +190,12 @@ func (storage *PublishedStorage) Remove(path string) error {
 	}
 	_, err := storage.s3.DeleteObject(params)
 	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok {
+			if aerr.Code() == s3.ErrCodeNoSuchBucket {
+				// ignore 'no such bucket' errors on removal
+				return nil
+			}
+		}
 		return errors.Wrap(err, fmt.Sprintf("error deleting %s from %s", path, storage))
 	}
 
@@ -206,6 +212,12 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 
 	filelist, _, err := storage.internalFilelist(path, false)
 	if err != nil {
+		if aerr, ok := errors.Cause(err).(awserr.Error); ok {
+			if aerr.Code() == s3.ErrCodeNoSuchBucket {
+				// ignore 'no such bucket' errors on removal
+				return nil
+			}
+		}
 		return err
 	}
 
@@ -263,10 +275,10 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, baseName)
+	relPath := filepath.Join(publishedDirectory, fileName)
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	if storage.pathCache == nil {
@@ -356,7 +368,7 @@ func (storage *PublishedStorage) internalFilelist(prefix string, hidePlusWorkaro
 	})
 
 	if err != nil {
-		return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
+		return nil, nil, errors.WithMessagef(err, "error listing under prefix %s in %s: %s", prefix, storage, err)
 	}
 
 	return paths, md5s, nil

s3/public_test.go

@@ -3,6 +3,7 @@ package s3
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	. "gopkg.in/check.v1"
@@ -17,6 +18,7 @@ import (
 type PublishedStorageSuite struct {
 	srv                      *Server
 	storage, prefixedStorage *PublishedStorage
+	noSuchBucketStorage      *PublishedStorage
 }
 
 var _ = Suite(&PublishedStorageSuite{})
@@ -31,6 +33,8 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	c.Assert(err, IsNil)
 	s.prefixedStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "test", "", "lala", "", "", false, true, false, false)
 	c.Assert(err, IsNil)
+	s.noSuchBucketStorage, err = NewPublishedStorage("aa", "bb", "", "test-1", s.srv.URL(), "no-bucket", "", "", "", "", false, true, false, false)
+	c.Assert(err, IsNil)
 
 	_, err = s.storage.s3.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String("test")})
 	c.Assert(err, IsNil)
@@ -169,6 +173,11 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	s.AssertNoFile(c, "lala/xyz")
 }
 
+func (s *PublishedStorageSuite) TestRemoveNoSuchBucket(c *C) {
+	err := s.noSuchBucketStorage.Remove("a/b")
+	c.Check(err, IsNil)
+}
+
 func (s *PublishedStorageSuite) TestRemovePlusWorkaround(c *C) {
 	s.storage.plusWorkaround = true
 
@@ -217,6 +226,11 @@ func (s *PublishedStorageSuite) TestRemoveDirsPlusWorkaround(c *C) {
 	c.Check(list, DeepEquals, []string{"a", "b", "c", "lala/a", "lala/b", "lala/c", "testa"})
 }
 
+func (s *PublishedStorageSuite) TestRemoveDirsNoSuchBucket(c *C) {
+	err := s.noSuchBucketStorage.RemoveDirs("a/b", nil)
+	c.Check(err, IsNil)
+}
+
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	c.Skip("copy not available in s3test")
 }
@@ -236,10 +250,18 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
 
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
 	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
 	c.Assert(err, IsNil)
 	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
 	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
 
 	// first link from pool
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
@@ -279,6 +301,11 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 
 	c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
 
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
 }
 
 func (s *PublishedStorageSuite) TestSymLink(c *C) {

s3/server_test.go

@@ -73,7 +73,6 @@ type Server struct {
 type bucket struct {
 	name    string
 	acl     string
-	ctime   time.Time
 	objects map[string]*object
 }
 

swift/public.go

@@ -193,10 +193,10 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 // sourcePath is filepath to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
-	relPath := filepath.Join(publishedDirectory, baseName)
+	relPath := filepath.Join(publishedDirectory, fileName)
 	poolPath := filepath.Join(storage.prefix, relPath)
 
 	var (

swift/public_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"time"
 
@@ -157,10 +158,18 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
 
+	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
+	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	c.Assert(err, IsNil)
+	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+
 	src1, err := pool.Import(tmpFile1, "mars-invaders_1.03.deb", &cksum1, true, cs)
 	c.Assert(err, IsNil)
 	src2, err := pool.Import(tmpFile2, "mars-invaders_1.03.deb", &cksum2, true, cs)
 	c.Assert(err, IsNil)
+	src3, err := pool.Import(tmpFile3, "netboot/boot.img.gz", &cksum3, true, cs)
+	c.Assert(err, IsNil)
 
 	// first link from pool
 	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
@@ -193,6 +202,14 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	data, err = s.storage.conn.ObjectGetBytes("test", "pool/main/m/mars-invaders/mars-invaders_1.03.deb")
 	c.Check(err, IsNil)
 	c.Check(data, DeepEquals, []byte("Spam"))
+
+	// link from pool with nested file name
+	err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
+	c.Check(err, IsNil)
+
+	data, err = s.storage.conn.ObjectGetBytes("test", "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("Contents"))
 }
 
 func (s *PublishedStorageSuite) TestSymLink(c *C) {

system/gpg-gen-key

@@ -10,4 +10,4 @@ Expire-Date: 0
 %secring aptly.sec
 # Do a commit here, so that we can later print "done" :-)
 %commit
-%echo done
+%echo done

system/lib.py

@@ -50,6 +50,34 @@ class FileHTTPServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
         pass
 
 
+class GPGFinder(object):
+    """
+    GnuPG binary discovery.
+    """
+
+    def __init__(self):
+        self.gpg1 = self.find_gpg(["gpg1", "gpg"], "gpg (GnuPG) 1.")
+        self.gpg2 = self.find_gpg(["gpg2", "gpg"], "gpg (GnuPG) 2.")
+
+        self.gpg = self.gpg1
+        if self.gpg is None:
+            self.gpg = self.gpg2
+
+        if self.gpg is None:
+            raise Exception("GnuPG binary wasn't found")
+
+    def find_gpg(self, executables, expected_version):
+        for executable in executables:
+            try:
+                output = subprocess.check_output([executable, "--version"])
+                if expected_version in output:
+                    return executable
+            except Exception:
+                pass
+
+        return None
+
+
 class BaseTest(object):
     """
     Base class for all tests.
@@ -62,6 +90,8 @@ class BaseTest(object):
     fixtureGpg = False
     fixtureWebServer = False
     requiresFTP = False
+    requiresGPG1 = False
+    requiresGPG2 = False
 
     expectedCode = 0
     configFile = {
@@ -84,6 +114,7 @@ class BaseTest(object):
     fixtureDBDir = os.path.join(os.environ["HOME"], "aptly-fixture-db")
     fixturePoolDir = os.path.join(os.environ["HOME"], "aptly-fixture-pool")
     fixtureGpgKeys = ["debian-archive-keyring.gpg",
+                      "ubuntu-archive-keyring.gpg",
                       "launchpad.key",
                       "flat.key",
                       "pagerduty.key",
@@ -94,6 +125,8 @@ class BaseTest(object):
 
     captureResults = False
 
+    gpgFinder = GPGFinder()
+
     def test(self):
         self.prepare()
         self.run()
@@ -109,6 +142,10 @@ class BaseTest(object):
 
     def prepare_default_config(self):
         cfg = self.configFile.copy()
+        if self.requiresGPG1:
+            cfg["gpgProvider"] = "gpg1"
+        elif self.requiresGPG2:
+            cfg["gpgProvider"] = "gpg2"
         cfg.update(**self.configOverride)
         f = open(os.path.join(os.environ["HOME"], ".aptly.conf"), "w")
         f.write(json.dumps(cfg))
@@ -121,6 +158,10 @@ class BaseTest(object):
             return False
         if self.requiresFTP and os.environ.get('NO_FTP_ACCESS', '') == 'yes':
             return False
+        if self.requiresGPG1 and self.gpgFinder.gpg1 is None:
+            return False
+        if self.requiresGPG2 and self.gpgFinder.gpg2 is None:
+            return False
 
         return True
 
@@ -140,8 +181,13 @@ class BaseTest(object):
             self.webServerUrl = self.start_webserver(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)),
                                                      self.fixtureWebServer))
 
+        if self.requiresGPG2:
+            self.run_cmd([
+                self.gpgFinder.gpg2, "--import",
+                os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files") + "/aptly.sec"], expected_code=None)
+
         if self.fixtureGpg:
-            self.run_cmd(["gpg", "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] +
+            self.run_cmd([self.gpgFinder.gpg, "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] +
                          [os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", key) for key in self.fixtureGpgKeys])
 
         if hasattr(self, "fixtureCmds"):
@@ -175,8 +221,9 @@ class BaseTest(object):
         try:
             proc = self._start_process(command, stdout=subprocess.PIPE)
             output, _ = proc.communicate()
-            if proc.returncode != expected_code:
+            if expected_code is not None:
-                raise Exception("exit code %d != %d (output: %s)" % (proc.returncode, expected_code, output))
+                if proc.returncode != expected_code:
+                    raise Exception("exit code %d != %d (output: %s)" % (proc.returncode, expected_code, output))
             return output
         except Exception, e:
             raise Exception("Running command %s failed: %s" % (command, str(e)))

system/t03_help/MainHelpTest_gold

@@ -19,5 +19,5 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
 

system/t03_help/MainTest_gold

@@ -27,5 +27,5 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
 ERROR: unable to parse command

system/t03_help/MirrorCreateHelpTest_gold

@@ -25,9 +25,10 @@ Options:
   -filter-with-deps: when filtering, include dependencies of matching packages as well
   -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
   -force-components: (only with component list) skip check that requested components are listed in Release file
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
   -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
+  -with-installer: download additional not packaged installer files
   -with-sources: download source packages in addition to binary packages
   -with-udebs: download .udeb packages (Debian installer support)
 

system/t03_help/MirrorCreateTest_gold

@@ -16,9 +16,10 @@ Options:
   -filter-with-deps: when filtering, include dependencies of matching packages as well
   -force-architectures: (only with architecture list) skip check that requested architectures are listed in Release file
   -force-components: (only with component list) skip check that requested components are listed in Release file
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
   -ignore-signatures: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
+  -with-installer: download additional not packaged installer files
   -with-sources: download source packages in addition to binary packages
   -with-udebs: download .udeb packages (Debian installer support)
 ERROR: unable to parse command

system/t03_help/MirrorHelpTest_gold

@@ -23,4 +23,4 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)

system/t03_help/MirrorTest_gold

@@ -23,5 +23,5 @@ Options:
   -dep-follow-source: when processing dependencies, follow from binary to Source packages
   -dep-follow-suggests: when processing dependencies, follow Suggests
   -dep-verbose-resolve: when processing dependencies, print detailed logs
-  -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+  -gpg-provider="": PGP implementation ("gpg", "gpg1", "gpg2" for external gpg or "internal" for Go internal implementation)
 ERROR: unable to parse command