Merge pull request #847 from smira/fix-repo-name

Fix repo name in release script

.gitignore

@@ -22,8 +22,7 @@ _testmain.go
 *.exe
 *.test
 
-coverage.html
+coverage.txt
-coverage*.out
 
 *.pyc
 
@@ -36,3 +35,9 @@ man/aptly.1.ronn
 .goxc.local.json
 
 system/env/
+
+# created by make build for release artifacts
+build/
+
+pgp/keyrings/aptly2*.gpg
+pgp/keyrings/aptly2*.gpg~

.golangci.yml

@@ -0,0 +1,22 @@
+run:
+  tests: false
+
+
+linters:
+  enable-all: false
+  disable-all: true
+  enable:
+   - govet
+   - golint
+   - gofmt
+   - deadcode
+   - goimports
+   - misspell
+   - ineffassign
+   - staticcheck
+   - varcheck
+   - structcheck
+   - maligned
+   - vetshadow
+   - goconst
+   - interfacer

.travis.yml

@@ -3,25 +3,42 @@ sudo: required
 
 language: go
 
-go:
+go_import_path: github.com/aptly-dev/aptly
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
-  - master
-
-go_import_path: github.com/smira/aptly
 
 addons:
   apt:
     packages:
     - python-virtualenv
     - graphviz
+    - gnupg2
+    - gpgv2
 
 env:
   global:
-    - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
+    - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
-    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
+    - NO_FTP_ACCESS: "yes"
+    - BOTO_CONFIG: /dev/null
+
+matrix:
+  allow_failures:
+    - go: master
+      env: RUN_LONG_TESTS=no
+  fast_finish: true
+  include:
+    - go: 1.10.x
+      env: RUN_LONG_TESTS=no
+    - go: 1.11.x
+      env: RUN_LONG_TESTS=yes
+    - go: 1.12.x
+      env:
+        - RUN_LONG_TESTS=yes
+        - DEPLOY_BINARIES=yes
+        - APTLY_USER=aptly
+        - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
+    - go: master
+      env: RUN_LONG_TESTS=no
+
 before_install:
   - virtualenv system/env
   - . system/env/bin/activate
@@ -29,14 +46,12 @@ before_install:
   - pip install -U pip setuptools
   - pip install -r system/requirements.txt
   - make version
+
 install:
   - make prepare
 
-script: make travis
+after_success:
-
+  - bash <(curl -s https://codecov.io/bash)
-matrix:
-    allow_failures:
-        - go: master
 
 notifications:
   webhooks:
@@ -45,3 +60,41 @@ notifications:
     on_success: change  # options: [always|never|change] default: always
     on_failure: always  # options: [always|never|change] default: always
     on_start: false     # default: false
+
+before_deploy:
+  - make release
+
+deploy:
+  - provider: releases
+    api_key:
+      secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
+    file_glob: true
+    file: build/*
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: s3
+    access_key_id:
+      secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
+    secret_access_key:
+      secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
+    bucket: aptly-nightly
+    skip_cleanup: true
+    acl: public-read
+    local_dir: build
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh nightly
+    skip_cleanup: true
+    on:
+      branch: master
+      condition: "$DEPLOY_BINARIES = yes"
+  - provider: script
+    script: bash upload-artifacts.sh release
+    skip_cleanup: true
+    on:
+      tags: true
+      condition: "$DEPLOY_BINARIES = yes"

AUTHORS

@@ -28,3 +28,10 @@ List of contributors, in chronological order:
 * Charles Hsu (https://github.com/charz)
 * Clemens Rabe (https://github.com/seeraven)
 * TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)
+* Petr Jediny (https://github.com/pjediny)
+* Maximilian Stein (https://github.com/steinymity)
+* Strajan Sebastian (https://github.com/strajansebastian)
+* Artem Smirnov (https://github.com/urpylka)
+* William Manley (https://github.com/wmanley)

CONTRIBUTING.md

@@ -15,7 +15,7 @@ Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info)
 
 ### List of Repositories
 
-* [smira/aptly](https://github.com/smira/aptly) - aptly source code, functional tests, man page
+* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
 * [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
@@ -24,15 +24,15 @@ Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info)
 
 ### Reporting Bugs
 
-1. Please search for similar bug report in [issue tracker](https://github.com/smira/aptly/issues)
+1. Please search for similar bug report in [issue tracker](https://github.com/aptly-dev/aptly/issues)
 2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
 3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
 4. If bug is reproducible with specific package, please provide link to package file.
-5. Open issue at [GitHub](https://github.com/smira/aptly/issues)
+5. Open issue at [GitHub](https://github.com/aptly-dev/aptly/issues)
 
 ### Suggesting Enhancements
 
-1. Please search [issue tracker](https://github.com/smira/aptly/issues) for similar feature requests.
+1. Please search [issue tracker](https://github.com/aptly-dev/aptly/issues) for similar feature requests.
 2. Describe why enhancement is important to you.
 3. Include any additional details or implementation details.
 
@@ -45,7 +45,7 @@ There are two kinds of documentation:
 
 Core content is mostly the same, but website contains more information, tutorials, examples.
 
-If you want to update `man` page, please open PR to [main aptly repo](https://github.com/smira/aptly),
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/aptly-dev/aptly),
 details in [man page](#man-page) section.
 
 If you want to update website, please follow steps below:
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
 We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
 general fixes, clarifications, misspellings, grammar mistakes!
 
-### Your Fist Code Contribution
+### Your First Code Contribution
 
 Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
 following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -86,9 +86,9 @@ to prepend it or to skip this test if you're security conscious.
 
 As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
 
-    mkdir -p ~/go/src/github.com/smira
+    mkdir -p ~/go/src/github.com/aptly-dev
-    cd ~/go/src/github.com/smira
+    cd ~/go/src/github.com/aptly-dev
-    git clone git@github.com:smira/aptly.git
+    git clone git@github.com:aptly-dev/aptly.git
     cd aptly
 
 For main repo under your GitHub user and add it as another Git remote:
@@ -184,9 +184,8 @@ Style checks could be run with:
 
     make check
 
-aptly is using [gometalinter](https://github.com/alecthomas/gometalinter) to run style checks on Go code. Configuration
+aptly is using [golangci-lint](https://github.com/golangci/golangci-lint) to run style checks on Go code. Configuration
-for the linter could be found in [linter.json](linter.json) file. Running linters might take considerable amount of time
+for the linter could be found in [.golangci.yml](.golangci.yml) file.
-unfortunately, but usually warning reported by linters hint at real code issues.
 
 Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
 
@@ -206,12 +205,13 @@ template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any comman
 final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
 verification.
 
-Man page is built with small helper [_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+Man page is built with small helper [\_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
 and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
 
-### Bash Completion
+### Bash and Zsh Completion
 
-Bash completion for aptly resides in the same repo under in [bash_completion.d/aptly](bash_completion.d/aptly). It's all hand-crafted.
+Bash and Zsh completion for aptly reside in the same repo under in [completion.d/aptly](completion.d/aptly) and
+[completion.d/\_aptly](completion.d/_aptly), respectively. It's all hand-crafted.
 When new option or command is introduced, bash completion should be updated to reflect that change.
 
 When aptly package is being built, it automatically pulls bash completion and man page into the package.

Gopkg.lock

@@ -1,191 +1,404 @@
-memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
+# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
+
 
 [[projects]]
+  digest = "1:e8777c437e157121465a08c05304d5847ae70b5683ef4afeb723b3c9e5e3bc67"
   name = "github.com/AlekSi/pointer"
   packages = ["."]
+  pruneopts = ""
   revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
   version = "v1.0.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:218a66570875d243938fff08f17ec03e79b18c138d0845b2428bcb4929ffa4fe"
   name = "github.com/DisposaBoy/JsonConfigReader"
   packages = ["."]
+  pruneopts = ""
   revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
 
 [[projects]]
+  digest = "1:658ab137074ef3d1216e01cbe166eb171a1bd77daeae0b686ab1ae184a9a5ec9"
   name = "github.com/awalterschulze/gographviz"
-  packages = [".","ast","parser","scanner","token"]
+  packages = [
+    ".",
+    "ast",
+    "parser",
+    "scanner",
+    "token"
+  ]
+  pruneopts = ""
   revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
   version = "v1.0"
 
 [[projects]]
+  digest = "1:a72e35a51b628d148241720897de644fd6d3cea45d8489dae9d373ced5dfc302"
   name = "github.com/aws/aws-sdk-go"
-  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
+  packages = [
-  revision = "2db5849d2939d93075d911138309a83235032bea"
+    "aws",
-  version = "v1.8.0"
+    "aws/awserr",
+    "aws/awsutil",
+    "aws/client",
+    "aws/client/metadata",
+    "aws/corehandlers",
+    "aws/credentials",
+    "aws/credentials/ec2rolecreds",
+    "aws/credentials/endpointcreds",
+    "aws/credentials/stscreds",
+    "aws/defaults",
+    "aws/ec2metadata",
+    "aws/endpoints",
+    "aws/request",
+    "aws/session",
+    "aws/signer/v4",
+    "internal/sdkio",
+    "internal/sdkrand",
+    "internal/shareddefaults",
+    "private/protocol",
+    "private/protocol/query",
+    "private/protocol/query/queryutil",
+    "private/protocol/rest",
+    "private/protocol/restxml",
+    "private/protocol/xml/xmlutil",
+    "service/s3",
+    "service/sts"
+  ]
+  pruneopts = ""
+  revision = "a72204b9bf8d48230ee0fe8995613b394c66f2da"
+  version = "v1.13.31"
 
 [[projects]]
+  digest = "1:b9922c7da8a89c973758c03bde497017567f785b49bd253402a3587abec1e53d"
   name = "github.com/cheggaaa/pb"
   packages = ["."]
+  pruneopts = ""
   revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
   version = "v1.0.10"
 
 [[projects]]
-  name = "github.com/gin-gonic/gin"
+  branch = "master"
-  packages = [".","binding","render"]
+  digest = "1:1120f960f5c334f0f94bad29eefaf73d52d226893369693686148f66c1993f15"
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
+  name = "github.com/gin-contrib/sse"
+  packages = ["."]
+  pruneopts = ""
+  revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
 
 [[projects]]
+  digest = "1:348ceb76f2ac958e541e4ba3190484b68df28c38ac9720ed4ef8d36af69ce52e"
+  name = "github.com/gin-gonic/gin"
+  packages = [
+    ".",
+    "binding",
+    "render"
+  ]
+  pruneopts = ""
+  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
+  version = "v1.2"
+
+[[projects]]
+  digest = "1:fbed9ba4076145ae05ef7deeb14d49565e112d6c283a8c4304fa3b7be785fa5e"
   name = "github.com/go-ini/ini"
   packages = ["."]
+  pruneopts = ""
   revision = "1730955e3146956d6a087861380f9b4667ed5071"
   version = "v1.26.0"
 
 [[projects]]
   branch = "master"
+  digest = "1:27854310d59099f8dcc61dd8af4a69f0a3597f001154b2fb4d1c41baf2e31ec1"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  pruneopts = ""
+  revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
+
+[[projects]]
+  branch = "master"
+  digest = "1:09307dfb1aa3f49a2bf869dcfa4c6c06ecd3c207221bd1c1a1141f0e51f209eb"
   name = "github.com/golang/snappy"
   packages = ["."]
+  pruneopts = ""
   revision = "553a641470496b2327abcac10b36396bd98e45c9"
 
 [[projects]]
-  branch = "master"
+  digest = "1:6ee50e0ace655f26787c120dbce51b3185c470f475c9e57d0dcada8ebb115004"
   name = "github.com/h2non/filetype"
   packages = ["matchers"]
-  revision = "0df83c38d14ff5f653d419d480eaac286ccbc823"
+  pruneopts = ""
+  revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb"
+  version = "v1.0.5"
 
 [[projects]]
   branch = "master"
+  digest = "1:64b78d98b8956492576911baf6a1e3499816d4575e485d12792e4abe7d8b6c46"
   name = "github.com/jlaffaye/ftp"
   packages = ["."]
-  revision = "7b85eb4638a2c0473acefcfb929a98f879c15c86"
+  pruneopts = ""
+  revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
 
 [[projects]]
+  digest = "1:6f49eae0c1e5dab1dafafee34b207aeb7a42303105960944828c2079b92fc88e"
   name = "github.com/jmespath/go-jmespath"
   packages = ["."]
-  revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9"
+  pruneopts = ""
-  version = "0.2.2"
+  revision = "0b12d6b5"
 
 [[projects]]
-  name = "github.com/julienschmidt/httprouter"
+  branch = "master"
+  digest = "1:c728183dd470c8bb2d1775edc641a81ec25f5e3804b8b0536e99f71df17a13a6"
+  name = "github.com/kjk/lzma"
   packages = ["."]
-  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
+  pruneopts = ""
-  version = "v1.1"
+  revision = "3fd93898850d5252457e48c1b3d5e1510597280b"
 
 [[projects]]
+  digest = "1:78229b46ddb7434f881390029bd1af7661294af31f6802e0e1bedaad4ab0af3c"
+  name = "github.com/mattn/go-isatty"
+  packages = ["."]
+  pruneopts = ""
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
+
+[[projects]]
+  digest = "1:81e673df85e765593a863f67cba4544cf40e8919590f04d67664940786c2b61a"
   name = "github.com/mattn/go-runewidth"
   packages = ["."]
+  pruneopts = ""
   revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
   version = "v0.0.2"
 
 [[projects]]
+  digest = "1:fb8502ed69803c4cd97b1def119db73869ec2f7e2b408eb9c1b8276df6f05b3e"
   name = "github.com/mattn/go-shellwords"
   packages = ["."]
+  pruneopts = ""
   revision = "005a0944d84452842197c2108bd9168ced206f78"
   version = "v1.0.2"
 
 [[projects]]
   branch = "master"
+  digest = "1:3ba28ef4fbbf8d099c6227b698a6ceae28f8a071e12f06839c12c4c0003b7554"
   name = "github.com/mkrautz/goar"
   packages = ["."]
+  pruneopts = ""
   revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
 
 [[projects]]
   branch = "master"
+  digest = "1:d33ce379780d7c43405b9251289493cabada82f6bf9ab35eea6915d04f6ac8e0"
   name = "github.com/mxk/go-flowrate"
   packages = ["flowrate"]
+  pruneopts = ""
   revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
 
 [[projects]]
   branch = "master"
+  digest = "1:428d8af27f534ed06d783b03d477124796de06aa86402777cd2b494c64278da5"
   name = "github.com/ncw/swift"
-  packages = [".","swifttest"]
+  packages = [
+    ".",
+    "swifttest"
+  ]
+  pruneopts = ""
   revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
 
 [[projects]]
+  branch = "master"
+  digest = "1:beeb9206cc21cfeb113066c3dcf4bbb0ba304d73dd441f3244721566f51f44e6"
+  name = "github.com/pborman/uuid"
+  packages = ["."]
+  pruneopts = ""
+  revision = "c65b2f87fee37d1c7854c9164a450713c28d50cd"
+
+[[projects]]
+  digest = "1:1d7e1867c49a6dd9856598ef7c3123604ea3daabf5b83f303ff457bcbc410b1d"
   name = "github.com/pkg/errors"
   packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  pruneopts = ""
-  version = "v0.8.0"
+  revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
+  version = "v0.8.1"
 
 [[projects]]
   branch = "master"
+  digest = "1:c6e71685a14409c3a1fdbecef044ee6447c3887de7c2bb025ee12f9a8a368960"
   name = "github.com/smira/commander"
   packages = ["."]
+  pruneopts = ""
   revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
 
 [[projects]]
   branch = "master"
+  digest = "1:a75eb2e6e718988505f704d651eeb4734f0f0b580f761aa58b4a5b0afe585091"
   name = "github.com/smira/flag"
   packages = ["."]
-  revision = "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107"
+  pruneopts = ""
+  revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
 
 [[projects]]
   branch = "master"
+  digest = "1:cd3b8057abbb453404cbef569951a7359986eedfc0a887f9bc5efff8d02e1760"
   name = "github.com/smira/go-aws-auth"
   packages = ["."]
-  revision = "0070896e9d7f4f9f2d558532b2d896ce2239992a"
+  pruneopts = ""
+  revision = "8b73995fd8d1d82519c7037ee243a497552cd54d"
 
 [[projects]]
   branch = "master"
+  digest = "1:7dd59b3536a07651baf394466da3b8a20bd3ed9ce1984c7583fdf2412fdcfea9"
   name = "github.com/smira/go-ftp-protocol"
   packages = ["protocol"]
+  pruneopts = ""
   revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
 
 [[projects]]
   branch = "master"
-  name = "github.com/smira/go-uuid"
+  digest = "1:55fe1086100b305a5562e9f1e6825ef97c49e82331cb0f055b2069f7c7ebd469"
-  packages = ["uuid"]
-  revision = "ed3ca8a15a931b141440a7e98e4f716eec255f7d"
-
-[[projects]]
-  branch = "master"
   name = "github.com/smira/go-xz"
   packages = ["."]
+  pruneopts = ""
   revision = "0c531f070014e218b21f3cfca801cc992d52726d"
 
 [[projects]]
   branch = "master"
-  name = "github.com/smira/lzma"
+  digest = "1:175bc8d87d54849b9b9fc6cd473c5830e090fbf3c2f0ca71c7642a697e5d9237"
-  packages = ["."]
-  revision = "7f0af6269940baa2c938fabe73e0d7ba41205683"
-
-[[projects]]
-  branch = "master"
   name = "github.com/syndtr/goleveldb"
-  packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
+  packages = [
-  revision = "3c5717caf1475fd25964109a0fc640bd150fce43"
+    "leveldb",
+    "leveldb/cache",
+    "leveldb/comparer",
+    "leveldb/errors",
+    "leveldb/filter",
+    "leveldb/iterator",
+    "leveldb/journal",
+    "leveldb/memdb",
+    "leveldb/opt",
+    "leveldb/storage",
+    "leveldb/table",
+    "leveldb/util"
+  ]
+  pruneopts = ""
+  revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
 
 [[projects]]
+  digest = "1:b9bca27d5fbe4ad1a1802706629580ce923bb14255c51375480dff1bebcbb8b2"
   name = "github.com/ugorji/go"
   packages = ["codec"]
+  pruneopts = ""
   revision = "71c2886f5a673a35f909803f38ece5810165097b"
 
 [[projects]]
   branch = "master"
+  digest = "1:2d8078b329a80bf4b71537dcd30853ff89fcafb611c94b2dfdb3d43ef1f23a1a"
   name = "github.com/wsxiaoys/terminal"
   packages = ["color"]
+  pruneopts = ""
   revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
 
 [[projects]]
   branch = "master"
+  digest = "1:51aff2b69272cb95713b74675c0817e0a0caadc20337a8d9115bc68bf1105280"
   name = "golang.org/x/crypto"
-  packages = ["cast5","openpgp","openpgp/armor","openpgp/clearsign","openpgp/elgamal","openpgp/errors","openpgp/packet","openpgp/s2k","ssh/terminal"]
+  packages = [
-  revision = "459e26527287adbc2adcc5d0d49abff9a5f315a7"
+    "cast5",
+    "openpgp",
+    "openpgp/armor",
+    "openpgp/clearsign",
+    "openpgp/elgamal",
+    "openpgp/errors",
+    "openpgp/packet",
+    "openpgp/s2k",
+    "ssh/terminal"
+  ]
+  pruneopts = ""
+  revision = "b2aa35443fbc700ab74c586ae79b81c171851023"
 
 [[projects]]
   branch = "master"
+  digest = "1:267aa6124260d0e48d1ebf8bb84dce97caa133406b3caabc076cfc7a9e85ffe4"
   name = "golang.org/x/sys"
-  packages = ["unix"]
+  packages = [
-  revision = "99f16d856c9836c42d24e7ab64ea72916925fa97"
+    "unix",
+    "windows"
+  ]
+  pruneopts = ""
+  revision = "1d206c9fa8975fb4cf00df1dc8bf3283dc24ba0e"
 
 [[projects]]
   branch = "v1"
+  digest = "1:e75566abfb876e81f00290ec153ff994c33bf8886134c1a38a9a9df5c15a2045"
   name = "gopkg.in/check.v1"
   packages = ["."]
+  pruneopts = ""
   revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
 
 [[projects]]
+  digest = "1:dd549e360e5a8f982a28c2bcbe667307ceffe538ed9afc7c965524f1ac285b3f"
+  name = "gopkg.in/go-playground/validator.v8"
+  packages = ["."]
+  pruneopts = ""
+  revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
+  version = "v8.18.2"
+
+[[projects]]
+  digest = "1:0e886d5a5845f58255fb57bcb4a6aae7e1f9dd7c53defe7b87d57dd4cd127545"
   name = "gopkg.in/h2non/filetype.v1"
   packages = ["types"]
+  pruneopts = ""
   revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
   version = "v1.0.1"
+
+[[projects]]
+  branch = "v2"
+  digest = "1:81314a486195626940617e43740b4fa073f265b0715c9f54ce2027fee1cb5f61"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  pruneopts = ""
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"
+
+[solve-meta]
+  analyzer-name = "dep"
+  analyzer-version = 1
+  input-imports = [
+    "github.com/AlekSi/pointer",
+    "github.com/DisposaBoy/JsonConfigReader",
+    "github.com/awalterschulze/gographviz",
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/awserr",
+    "github.com/aws/aws-sdk-go/aws/corehandlers",
+    "github.com/aws/aws-sdk-go/aws/credentials",
+    "github.com/aws/aws-sdk-go/aws/request",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/s3",
+    "github.com/cheggaaa/pb",
+    "github.com/gin-gonic/gin",
+    "github.com/h2non/filetype/matchers",
+    "github.com/kjk/lzma",
+    "github.com/mattn/go-shellwords",
+    "github.com/mkrautz/goar",
+    "github.com/mxk/go-flowrate/flowrate",
+    "github.com/ncw/swift",
+    "github.com/ncw/swift/swifttest",
+    "github.com/pborman/uuid",
+    "github.com/pkg/errors",
+    "github.com/smira/commander",
+    "github.com/smira/flag",
+    "github.com/smira/go-aws-auth",
+    "github.com/smira/go-ftp-protocol/protocol",
+    "github.com/smira/go-xz",
+    "github.com/syndtr/goleveldb/leveldb",
+    "github.com/syndtr/goleveldb/leveldb/filter",
+    "github.com/syndtr/goleveldb/leveldb/opt",
+    "github.com/syndtr/goleveldb/leveldb/storage",
+    "github.com/syndtr/goleveldb/leveldb/util",
+    "github.com/ugorji/go/codec",
+    "github.com/wsxiaoys/terminal/color",
+    "golang.org/x/crypto/openpgp",
+    "golang.org/x/crypto/openpgp/armor",
+    "golang.org/x/crypto/openpgp/clearsign",
+    "golang.org/x/crypto/openpgp/errors",
+    "golang.org/x/crypto/openpgp/packet",
+    "golang.org/x/crypto/ssh/terminal",
+    "golang.org/x/sys/unix",
+    "gopkg.in/check.v1"
+  ]
+  solver-name = "gps-cdcl"
+  solver-version = 1

Gopkg.toml

@@ -1,32 +1,28 @@
 
-[[dependencies]]
+[[override]]
-  name = "github.com/gin-gonic/gin"
-  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
-
-[[dependencies]]
   branch = "master"
   name = "github.com/mkrautz/goar"
 
-[[dependencies]]
+[[override]]
   branch = "master"
   name = "github.com/smira/go-uuid"
 
-[[dependencies]]
+[[override]]
   branch = "master"
   name = "github.com/smira/go-xz"
 
-[[dependencies]]
+[[override]]
   name = "github.com/ugorji/go"
   revision = "71c2886f5a673a35f909803f38ece5810165097b"
 
-[[dependencies]]
+[[override]]
   branch = "master"
   name = "golang.org/x/crypto"
 
-[[dependencies]]
+[[override]]
   branch = "master"
   name = "golang.org/x/sys"
 
-[[dependencies]]
+[[override]]
   branch = "v1"
   name = "gopkg.in/check.v1"

Makefile

@@ -1,80 +1,72 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
+ifdef TRAVIS_TAG
+	TAG=$(TRAVIS_TAG)
+else
+	TAG="$(shell git describe --tags)"
+endif
+VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
 PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python
 TESTS?=
 BINPATH?=$(GOPATH)/bin
+RUN_LONG_TESTS?=yes
 
-ifeq ($(GOVERSION), devel)
+all: test bench check system-test
-TRAVIS_TARGET=coveralls
-else
-TRAVIS_TARGET=test
-endif
-
-all: test check system-test
 
 prepare:
-	go get -u github.com/mattn/goveralls
+	curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.17.1
-	go get -u github.com/axw/gocov/gocov
-	go get -u golang.org/x/tools/cmd/cover
-	go get -u github.com/alecthomas/gometalinter
-	gometalinter --install
 
 dev:
 	go get -u github.com/golang/dep/...
 	go get -u github.com/laher/goxc
 
-coverage.out:
-	rm -f coverage.*.out
-	for i in $(PACKAGES); do go test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
-	echo "mode: count" > coverage.out
-	grep -v -h "mode: count" coverage.*.out >> coverage.out
-	rm -f coverage.*.out
-
-coverage: coverage.out
-	go tool cover -html=coverage.out
-	rm -f coverage.out
-
 check: system/env
-	if [ -x travis_wait ]; then \
+ifeq ($(RUN_LONG_TESTS), yes)
-		travis_wait gometalinter --config=linter.json ./...; \
+	golangci-lint run
-	else \
-		gometalinter --config=linter.json ./...; \
-	fi
 	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
+endif
 
 install:
 	go install -v -ldflags "-X main.Version=$(VERSION)"
 
 system/env: system/requirements.txt
+ifeq ($(RUN_LONG_TESTS), yes)
 	rm -rf system/env
 	virtualenv system/env
 	system/env/bin/pip install -r system/requirements.txt
+endif
 
 system-test: install system/env
+ifeq ($(RUN_LONG_TESTS), yes)
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	. system/env/bin/activate && APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
-
+endif
-travis: $(TRAVIS_TARGET) check system-test
 
 test:
-	go test -v `go list ./... | grep -v vendor/` -gocheck.v=true
+	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
 
-coveralls: coverage.out
+bench:
-	$(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+	go test -v ./deb -run=nothing -bench=. -benchmem
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-goxc:
+goxc: dev
 	rm -rf root/
-	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
+	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
 	cp man/aptly.1 root/usr/share/man/man1
-	cp bash_completion.d/aptly root/etc/bash_completion.d
+	cp completion.d/aptly root/etc/bash_completion.d/
+	cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
 	gzip root/usr/share/man/man1/aptly.1
-	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
+	goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
+
+release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
+release: goxc
+	rm -rf build/
+	mkdir -p build/
+	mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
 
 man:
 	make -C man
@@ -82,4 +74,4 @@ man:
 version:
 	@echo $(VERSION)
 
-.PHONY: coverage.out man version
+.PHONY: man version release goxc

README.rst

@@ -2,17 +2,17 @@
 aptly
 =====
 
-.. image:: https://api.travis-ci.org/smira/aptly.svg?branch=master
+.. image:: https://api.travis-ci.org/aptly-dev/aptly.svg?branch=master
-    :target: https://travis-ci.org/smira/aptly
+    :target: https://travis-ci.org/aptly-dev/aptly
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.svg?branch=master
+.. image:: https://codecov.io/gh/aptly-dev/aptly/branch/master/graph/badge.svg
-    :target: https://coveralls.io/r/smira/aptly?branch=master
+  :target: https://codecov.io/gh/aptly-dev/aptly
 
 .. image:: https://badges.gitter.im/Join Chat.svg
-    :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+    :target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
 
-.. image:: http://goreportcard.com/badge/smira/aptly
+.. image:: http://goreportcard.com/badge/aptly-dev/aptly
-    :target: http://goreportcard.com/report/smira/aptly
+    :target: http://goreportcard.com/report/aptly-dev/aptly
 
 Aptly is a swiss army knife for Debian repository management.
 
@@ -48,7 +48,7 @@ To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list
 
 And import key that is used to sign the release::
 
-    $ apt-key adv --keyserver keys.gnupg.net --recv-keys 9E3E53F19C7DE460
+    $ apt-key adv --keyserver pool.sks-keyservers.net --recv-keys ED75B5A4483DA07C
 
 After that you can install aptly as any other software package::
 
@@ -64,14 +64,14 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.10+ required)::
 
-    mkdir -p $GOPATH/src/github.com/smira/aptly
+    mkdir -p $GOPATH/src/github.com/aptly-dev/aptly
-    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
+    git clone https://github.com/aptly-dev/aptly $GOPATH/src/github.com/aptly-dev/aptly
-    cd $GOPATH/src/github.com/smira/aptly
+    cd $GOPATH/src/github.com/aptly-dev/aptly
     make install
 
-Binary would be installed to ```$GOPATH/bin/aptly``.
+Binary would be installed to ``$GOPATH/bin/aptly``.
 
 Contributing
 ------------
@@ -90,7 +90,7 @@ Vagrant:
 Docker:
 
 -    `Docker container <https://github.com/mikepurvis/aptly-docker>`_ with aptly inside by Mike Purvis
--    `Docker container <https://github.com/bryanhong/docker-aptly>`_ with aptly and nginx by Bryan Hong
+-    `Docker container <https://github.com/urpylka/docker-aptly>`_ with aptly and nginx by Artem Smirnov
 
 With configuration management systems:
 
@@ -109,6 +109,10 @@ CLI for aptly API:
 -   `Ruby aptly CLI/library <https://github.com/sepulworld/aptly_cli>`_ by Zane Williamson
 -   `Python aptly CLI (good for CI) <https://github.com/TimSusa/aptly_api_cli>`_ by Tim Susa
 
+GUI for aptly API:
+
+-   `Python aptly GUI (via pyqt5) <https://github.com/chnyda/python-aptly-gui>`_ by Cedric Hnyda
+
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar

_man/gen.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/smira/aptly/cmd"
+	"github.com/aptly-dev/aptly/cmd"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

api/api.go

@@ -6,10 +6,10 @@ import (
 	"sort"
 	"time"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/query"
 )
 
 // Lock order acquisition (canonical):
@@ -112,7 +112,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -120,7 +120,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 
@@ -137,7 +137,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
@@ -147,7 +147,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
-			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}

api/files.go

@@ -24,7 +24,7 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.Fail(400, fmt.Errorf("wrong dir"))
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
@@ -53,7 +53,7 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -70,13 +70,13 @@ func apiFilesUpload(c *gin.Context) {
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -86,7 +86,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +94,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 
@@ -138,9 +138,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 		} else {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 		}
 		return
 	}
@@ -156,7 +156,7 @@ func apiFilesDeleteDir(c *gin.Context) {
 
 	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -170,14 +170,14 @@ func apiFilesDeleteFile(c *gin.Context) {
 	}
 
 	if !verifyPath(c.Params.ByName("name")) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
 	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}

api/graph.go

@@ -8,8 +8,8 @@ import (
 	"os"
 	"os/exec"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/deb"
 )
 
 // GET /api/graph.:ext?layout=[vertical|horizontal(default)]
@@ -24,14 +24,14 @@ func apiGraph(c *gin.Context) {
 
 	factory := context.CollectionFactory()
 
-	factory.RemoteRepoCollection().RLock()
+	factory.RemoteRepoCollection().Lock()
-	defer factory.RemoteRepoCollection().RUnlock()
+	defer factory.RemoteRepoCollection().Unlock()
-	factory.LocalRepoCollection().RLock()
+	factory.LocalRepoCollection().Lock()
-	defer factory.LocalRepoCollection().RUnlock()
+	defer factory.LocalRepoCollection().Unlock()
-	factory.SnapshotCollection().RLock()
+	factory.SnapshotCollection().Lock()
-	defer factory.SnapshotCollection().RUnlock()
+	defer factory.SnapshotCollection().Unlock()
-	factory.PublishedRepoCollection().RLock()
+	factory.PublishedRepoCollection().Lock()
-	defer factory.PublishedRepoCollection().RUnlock()
+	defer factory.PublishedRepoCollection().Unlock()
 
 	graph, err := deb.BuildGraph(factory, layout)
 	if err != nil {
@@ -53,25 +53,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/packages.go

@@ -8,7 +8,7 @@ import (
 func apiPackagesShow(c *gin.Context) {
 	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 

api/publish.go

@@ -4,10 +4,10 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/pgp"
-	"github.com/smira/aptly/utils"
 )
 
 // SigningOptions is a shared between publish API GPG options structure
@@ -60,8 +60,8 @@ func apiPublishList(c *gin.Context) {
 	defer snapshotCollection.RUnlock()
 
 	collection := context.CollectionFactory().PublishedRepoCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	result := make([]*deb.PublishedRepo, 0, collection.Len())
 
@@ -77,7 +77,7 @@ func apiPublishList(c *gin.Context) {
 	})
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -104,20 +104,21 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		SkipContents         *bool
 		Architectures        []string
 		Signing              SigningOptions
+		AcquireByHash        *bool
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
 	if len(b.Sources) == 0 {
-		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
 		return
 	}
 
@@ -128,21 +129,21 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		var snapshot *deb.Snapshot
 
 		snapshotCollection := context.CollectionFactory().SnapshotCollection()
-		snapshotCollection.RLock()
+		snapshotCollection.Lock()
-		defer snapshotCollection.RUnlock()
+		defer snapshotCollection.Unlock()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 
 			snapshot, err = snapshotCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = snapshotCollection.LoadComplete(snapshot)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
@@ -152,27 +153,27 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		var localRepo *deb.LocalRepo
 
 		localCollection := context.CollectionFactory().LocalRepoCollection()
-		localCollection.RLock()
+		localCollection.Lock()
-		defer localCollection.RUnlock()
+		defer localCollection.Unlock()
 
 		for _, source := range b.Sources {
 			components = append(components, source.Component)
 
 			localRepo, err = localCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = localCollection.LoadComplete(localRepo)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 			}
 
 			sources = append(sources, localRepo)
 		}
 	} else {
-		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
 		return
 	}
 
@@ -182,7 +183,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 	if b.Origin != "" {
@@ -201,22 +202,26 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	duplicate := collection.CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
-		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		c.AbortWithError(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
 		return
 	}
 
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 
 	err = collection.Add(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
@@ -233,30 +238,32 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		ForceOverwrite bool
 		Signing        SigningOptions
 		SkipContents   *bool
+		SkipCleanup    *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
 		}
+		AcquireByHash *bool
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
 	// published.LoadComplete would touch local repo collection
 	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.RLock()
+	localRepoCollection.Lock()
-	defer localRepoCollection.RUnlock()
+	defer localRepoCollection.Unlock()
 
 	snapshotCollection := context.CollectionFactory().SnapshotCollection()
-	snapshotCollection.RLock()
+	snapshotCollection.Lock()
-	defer snapshotCollection.RUnlock()
+	defer snapshotCollection.Unlock()
 
 	collection := context.CollectionFactory().PublishedRepoCollection()
 	collection.Lock()
@@ -264,12 +271,12 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 	err = collection.LoadComplete(published, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
@@ -277,7 +284,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
-			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
 		updatedComponents = published.Components()
@@ -288,19 +295,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		publishedComponents := published.Components()
 		for _, snapshotInfo := range b.Snapshots {
 			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
-				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
 				return
 			}
 
 			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
-			if err != nil {
+			if err2 != nil {
-				c.Fail(404, err2)
+				c.AbortWithError(404, err2)
 				return
 			}
 
 			err2 = snapshotCollection.LoadComplete(snapshot)
 			if err2 != nil {
-				c.Fail(500, err2)
+				c.AbortWithError(500, err2)
 				return
 			}
 
@@ -308,7 +315,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			updatedComponents = append(updatedComponents, snapshotInfo.Component)
 		}
 	} else {
-		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
@@ -316,23 +323,29 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
 	err = collection.Update(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
-	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+	if b.SkipCleanup == nil || !*b.SkipCleanup {
-		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
-	if err != nil {
+			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		if err != nil {
-		return
+			c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+			return
+		}
 	}
 
 	c.JSON(200, published)
@@ -341,6 +354,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
 
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
@@ -348,17 +362,17 @@ func apiPublishDrop(c *gin.Context) {
 
 	// published.LoadComplete would touch local repo collection
 	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
-	localRepoCollection.RLock()
+	localRepoCollection.Lock()
-	defer localRepoCollection.RUnlock()
+	defer localRepoCollection.Unlock()
 
 	collection := context.CollectionFactory().PublishedRepoCollection()
 	collection.Lock()
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), force)
+		context.CollectionFactory(), context.Progress(), force, skipCleanup)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 

api/repos.go

@@ -5,11 +5,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/aptly-dev/aptly/query"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/aptly"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
-	"github.com/smira/aptly/utils"
 )
 
 // GET /api/repos
@@ -37,7 +38,7 @@ func apiReposCreate(c *gin.Context) {
 		DefaultComponent    string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -51,7 +52,7 @@ func apiReposCreate(c *gin.Context) {
 
 	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -66,7 +67,7 @@ func apiReposEdit(c *gin.Context) {
 		DefaultComponent    *string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -76,7 +77,7 @@ func apiReposEdit(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -92,7 +93,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -107,7 +108,7 @@ func apiReposShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -132,27 +133,27 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.ByLocalRepo(repo)
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.ByLocalRepoSource(repo)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = collection.Drop(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -162,18 +163,18 @@ func apiReposDrop(c *gin.Context) {
 // GET /api/repos/:name/packages
 func apiReposPackagesShow(c *gin.Context) {
 	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -186,7 +187,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		PackageRefs []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -196,19 +197,19 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -219,15 +220,15 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = cb(list, p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -236,7 +237,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 
@@ -276,7 +277,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -286,13 +287,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -301,6 +302,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	var (
 		sources                      []string
 		packageFiles, failedFiles    []string
+		otherFiles                   []string
 		processedFiles, failedFiles2 []string
 		reporter                     = &aptly.RecordingResultReporter{
 			Warnings:     []string{},
@@ -316,11 +318,11 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
 		return
 	}
 
@@ -328,8 +330,10 @@ func apiReposPackageFromDir(c *gin.Context) {
 		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
 		return
 	}
 
@@ -337,7 +341,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 
@@ -364,3 +368,77 @@ func apiReposPackageFromDir(c *gin.Context) {
 		"FailedFiles": failedFiles,
 	})
 }
+
+// POST /repos/:name/include/:dir/:file
+func apiReposIncludePackageFromFile(c *gin.Context) {
+	// redirect all work to dir method
+	apiReposIncludePackageFromDir(c)
+}
+
+// POST /repos/:name/include/:dir
+func apiReposIncludePackageFromDir(c *gin.Context) {
+	forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
+	noRemoveFiles := c.Request.URL.Query().Get("noRemoveFiles") == "1"
+	acceptUnsigned := c.Request.URL.Query().Get("acceptUnsigned") == "1"
+	ignoreSignature := c.Request.URL.Query().Get("ignoreSignature") == "1"
+
+	repoTemplateString := c.Params.ByName("name")
+
+	if !verifyDir(c) {
+		return
+	}
+
+	fileParam := c.Params.ByName("file")
+	if fileParam != "" && !verifyPath(fileParam) {
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	var (
+		err                       error
+		verifier                  = context.GetVerifier()
+		sources, changesFiles     []string
+		failedFiles, failedFiles2 []string
+		reporter                  = &aptly.RecordingResultReporter{
+			Warnings:     []string{},
+			AddedLines:   []string{},
+			RemovedLines: []string{},
+		}
+	)
+
+	if fileParam == "" {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"))}
+	} else {
+		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
+	}
+
+	localRepoCollection := context.CollectionFactory().LocalRepoCollection()
+	localRepoCollection.Lock()
+	defer localRepoCollection.Unlock()
+
+	changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
+	_, failedFiles2, err = deb.ImportChangesFiles(
+		changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
+		repoTemplateString, context.Progress(), localRepoCollection, context.CollectionFactory().PackageCollection(),
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(), nil, query.Parse)
+	failedFiles = append(failedFiles, failedFiles2...)
+
+	if err != nil {
+		c.AbortWithError(500, fmt.Errorf("unable to import changes files: %s", err))
+		return
+	}
+
+	if !noRemoveFiles {
+		// atempt to remove dir, if it fails, that's fine: probably it's not empty
+		os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
+	}
+
+	if failedFiles == nil {
+		failedFiles = []string{}
+	}
+
+	c.JSON(200, gin.H{
+		"Report":      reporter,
+		"FailedFiles": failedFiles,
+	})
+}

api/router.go

@@ -3,8 +3,8 @@ package api
 import (
 	"net/http"
 
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/gin-gonic/gin"
-	ctx "github.com/smira/aptly/context"
 )
 
 var context *ctx.AptlyContext
@@ -32,7 +32,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 			err = <-errCh
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 
@@ -40,7 +40,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 				requests <- dbRequest{releasedb, errCh}
 				err = <-errCh
 				if err != nil {
-					c.Fail(500, err)
+					c.AbortWithError(500, err)
 				}
 			}()
 
@@ -71,6 +71,9 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
 		root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
 
+		root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
+		root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
+
 		root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
 	}
 

api/snapshot.go

@@ -3,9 +3,9 @@ package api
 import (
 	"fmt"
 
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/gin-gonic/gin"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/deb"
 )
 
 // GET /api/snapshots
@@ -42,13 +42,13 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	collection := context.CollectionFactory().RemoteRepoCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	snapshotCollection := context.CollectionFactory().SnapshotCollection()
 	snapshotCollection.Lock()
@@ -56,25 +56,25 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = repo.CheckLock()
 	if err != nil {
-		c.Fail(409, err)
+		c.AbortWithError(409, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -84,7 +84,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -105,7 +105,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		PackageRefs     []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -124,13 +124,13 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 			return
 		}
 
 		err = snapshotCollection.LoadComplete(sources[i])
 		if err != nil {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}
@@ -144,15 +144,15 @@ func apiSnapshotsCreate(c *gin.Context) {
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = list.Add(p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -161,7 +161,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -181,13 +181,13 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	collection := context.CollectionFactory().LocalRepoCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	snapshotCollection := context.CollectionFactory().SnapshotCollection()
 	snapshotCollection.Lock()
@@ -195,19 +195,19 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -217,7 +217,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -236,7 +236,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -246,13 +246,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	_, err = collection.ByName(b.Name)
 	if err == nil {
-		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		c.AbortWithError(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
 		return
 	}
 
@@ -266,7 +266,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -276,18 +276,18 @@ func apiSnapshotsUpdate(c *gin.Context) {
 // GET /api/snapshots/:name
 func apiSnapshotsShow(c *gin.Context) {
 	collection := context.CollectionFactory().SnapshotCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -309,28 +309,28 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.BySnapshot(snapshot)
 
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop: snapshot is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = snapshotCollection.Drop(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -342,37 +342,37 @@ func apiSnapshotsDiff(c *gin.Context) {
 	onlyMatching := c.Request.URL.Query().Get("onlyMatching") == "1"
 
 	collection := context.CollectionFactory().SnapshotCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -392,18 +392,18 @@ func apiSnapshotsDiff(c *gin.Context) {
 // GET /api/snapshots/:name/packages
 func apiSnapshotsSearchPackages(c *gin.Context) {
 	collection := context.CollectionFactory().SnapshotCollection()
-	collection.RLock()
+	collection.Lock()
-	defer collection.RUnlock()
+	defer collection.Unlock()
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 

aptly-api.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository API
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/api/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
+
+[Install]
+WantedBy=multi-user.target

aptly.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=APT repository server
+After=network.target
+Documentation=man:aptly(1)
+Documentation=https://www.aptly.info/doc/commands/
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
+
+[Install]
+WantedBy=multi-user.target

aptly/interfaces.go

@@ -3,10 +3,11 @@
 package aptly
 
 import (
+	"context"
 	"io"
 	"os"
 
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // ReadSeekerCloser = ReadSeeker + Closer
@@ -23,7 +24,7 @@ type PackagePool interface {
 	//
 	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
 	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
-	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (bool, error)
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
 	// Import copies file into package pool
 	//
 	// - srcPath is full path to source file as it is now
@@ -68,11 +69,19 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory, baseName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
+	LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
 // FileSystemPublishedStorage is published storage on filesystem
@@ -116,11 +125,13 @@ type Progress interface {
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string) error
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
 	// GetProgress returns Progress object
 	GetProgress() Progress
+	// GetLength returns size by heading object with url
+	GetLength(ctx context.Context, url string) (int64, error)
 }
 
 // ChecksumStorage is stores checksums in some (persistent) storage

cmd/api_serve.go

@@ -7,9 +7,9 @@ import (
 	"net/url"
 	"os"
 
-	"github.com/smira/aptly/api"
+	"github.com/aptly-dev/aptly/api"
-	"github.com/smira/aptly/systemd/activation"
+	"github.com/aptly-dev/aptly/systemd/activation"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -58,7 +58,25 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-		os.Remove(file)
+
+		var stat os.FileInfo
+		stat, err = os.Stat(file)
+		shouldRemove := true
+
+		if err == nil && stat.Mode()&os.ModeSocket == os.ModeSocket {
+			shouldRemove = false
+		}
+
+		if err != nil && os.IsNotExist(err) {
+			shouldRemove = false
+		}
+
+		if shouldRemove {
+			err = os.Remove(file)
+			if err != nil {
+				fmt.Printf("Warning: error removing file %s: %s\n", file, err)
+			}
+		}
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)

cmd/cmd.go

@@ -8,8 +8,8 @@ import (
 	"text/template"
 	"time"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -119,7 +119,7 @@ package environment to new version.`,
 	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
-	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\" for external gpg or \"internal\" for Go internal implementation)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/context.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	ctx "github.com/smira/aptly/context"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/flag"
 )
 
@@ -29,3 +29,8 @@ func InitContext(flags *flag.FlagSet) error {
 
 	return err
 }
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/db_cleanup.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 )
 
@@ -59,6 +59,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading local repos:@|")
 	}
@@ -90,6 +92,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading snapshots:@|")
 	}
@@ -118,6 +122,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading published repositories:@|")
 	}
@@ -150,6 +156,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	context.CollectionFactory().Flush()
+
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := context.CollectionFactory().PackageCollection().AllPackageRefs()
@@ -192,6 +200,8 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	context.CollectionFactory().Flush()
+
 	// now, build a list of files that should be present in Repository (package pool)
 	context.Progress().ColoredPrintf("@{w!}Building list of files referenced by packages...@|")
 	referencedFiles := make([]string, 0, existingPackageRefs.Len())

cmd/db_recover.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 	"github.com/smira/commander"
 )
 

cmd/graph.go

@@ -12,8 +12,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 )
 

cmd/mirror.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"strings"
 
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/mirror_create.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -19,6 +19,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
+	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -36,7 +37,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs)
+		downloadSources, downloadUdebs, downloadInstaller)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -94,6 +95,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")

cmd/mirror_edit.go

@@ -3,7 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -25,16 +26,22 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	fetchMirror := false
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
 			repo.Filter = flag.Value.String()
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-installer":
+			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
 		}
 	})
 
@@ -51,8 +58,17 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
 
-		err = repo.Fetch(context.Downloader(), nil)
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
@@ -83,10 +99,14 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_list.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/mirror_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/mirror_show.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/mirror_update.go

@@ -2,15 +2,13 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"os/signal"
 	"strings"
 	"sync"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -56,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Downloading & parsing package files...\n")
-	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries)
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -113,17 +111,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	// Catch ^C
+	context.GoContextHandleSignals()
-	sigch := make(chan os.Signal)
-	signal.Notify(sigch, os.Interrupt)
-	defer signal.Stop(sigch)
-
-	abort := make(chan struct{})
-	go func() {
-		<-sigch
-		signal.Stop(sigch)
-		close(abort)
-	}()
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
@@ -148,7 +136,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		for idx := range queue {
 			select {
 			case downloadQueue <- idx:
-			case <-abort:
+			case <-context.Done():
 				return
 			}
 		}
@@ -181,6 +169,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 
 					// download file...
 					e = context.Downloader().DownloadWithChecksum(
+						context,
 						repo.PackageURL(task.File.DownloadURL()).String(),
 						task.TempDownPath,
 						&task.File.Checksums,
@@ -190,28 +179,20 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						pushError(e)
 						continue
 					}
-				case <-abort:
+
+					task.Done = true
+				case <-context.Done():
 					return
 				}
 			}
 		}()
 	}
 
-	// Wait for all downloads to finish
+	// Wait for all download goroutines to finish
 	wg.Wait()
 
-	select {
-	case <-abort:
-		return fmt.Errorf("unable to update: interrupted")
-	default:
-	}
-
 	context.Progress().ShutdownBar()
 
-	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
-	}
-
 	err = context.ReOpenDatabase()
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -221,11 +202,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	context.Progress().InitBar(int64(len(queue)), false)
 
 	for idx := range queue {
-
 		context.Progress().AddBar(1)
 
 		task := &queue[idx]
 
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
 		// and import it back to the pool
 		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
 		if err != nil {
@@ -237,16 +222,20 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 			additionalTask.File.PoolPath = task.File.PoolPath
 			additionalTask.File.Checksums = task.File.Checksums
 		}
-
-		select {
-		case <-abort:
-			return fmt.Errorf("unable to update: interrupted")
-		default:
-		}
 	}
 
 	context.Progress().ShutdownBar()
 
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
 	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {

cmd/package_search.go

@@ -3,8 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/package_show.go

@@ -5,9 +5,9 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -79,7 +79,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 	result := q.Query(context.CollectionFactory().PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false)
+		p.Stanza().WriteTo(w, p.IsSource, false, false)
 		w.Flush()
 		fmt.Printf("\n")
 

cmd/publish.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/publish_drop.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 
@@ -24,7 +24,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
+		context.CollectionFactory(), context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -50,6 +52,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_list.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/publish_repo.go

@@ -47,6 +47,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_show.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/publish_snapshot.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -137,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -227,6 +231,7 @@ Example:
 	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -105,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -151,6 +154,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -69,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -109,6 +112,7 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/repo_add.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -41,9 +41,9 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	var packageFiles, failedFiles []string
+	var packageFiles, otherFiles, failedFiles []string
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
@@ -55,6 +55,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)

cmd/repo_create.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_edit.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/AlekSi/pointer"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_include.go

@@ -1,16 +1,11 @@
 package cmd
 
 import (
-	"bytes"
 	"fmt"
-	"os"
-	"path/filepath"
-	"text/template"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
-	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -35,11 +30,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
 	ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
 	noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
-
+	repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
-	repoTemplate, err := template.New("repo").Parse(context.Flags().Lookup("repo").Value.Get().(string))
-	if err != nil {
-		return fmt.Errorf("error parsing -repo template: %s", err)
-	}
 
 	uploaders := (*deb.Uploaders)(nil)
 	uploadersFile := context.Flags().Lookup("uploaders-file").Value.Get().(string)
@@ -59,139 +50,15 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 	reporter := &aptly.ConsoleResultReporter{Progress: context.Progress()}
 
-	var changesFiles, failedFiles, processedFiles []string
+	var changesFiles, failedFiles, failedFiles2 []string
 
 	changesFiles, failedFiles = deb.CollectChangesFiles(args, reporter)
-
+	_, failedFiles2, err = deb.ImportChangesFiles(
-	for _, path := range changesFiles {
+		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplateString,
-		var changes *deb.Changes
+		context.Progress(), context.CollectionFactory().LocalRepoCollection(), context.CollectionFactory().PackageCollection(),
-
+		context.PackagePool(), context.CollectionFactory().ChecksumCollection(),
-		changes, err = deb.NewChanges(path)
+		uploaders, query.Parse)
-		if err != nil {
+	failedFiles = append(failedFiles, failedFiles2...)
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", path, err)
-			continue
-		}
-
-		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		err = changes.Prepare()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		repoName := &bytes.Buffer{}
-		err = repoTemplate.Execute(repoName, changes.Stanza)
-		if err != nil {
-			return fmt.Errorf("error applying template to repo: %s", err)
-		}
-
-		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
-
-		var repo *deb.LocalRepo
-		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		currentUploaders := uploaders
-		if repo.Uploaders != nil {
-			currentUploaders = repo.Uploaders
-			for i := range currentUploaders.Rules {
-				currentUploaders.Rules[i].CompiledCondition, err = query.Parse(currentUploaders.Rules[i].Condition)
-				if err != nil {
-					return fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
-				}
-			}
-		}
-
-		if currentUploaders != nil {
-			if err = currentUploaders.IsAllowed(changes); err != nil {
-				failedFiles = append(failedFiles, path)
-				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
-					changes.ChangesName, changes.SignatureKeys, err)
-				changes.Cleanup()
-				continue
-			}
-		}
-
-		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
-			return fmt.Errorf("unable to load repo: %s", err)
-		}
-
-		var list *deb.PackageList
-		list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
-		if err != nil {
-			return fmt.Errorf("unable to load packages: %s", err)
-		}
-
-		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
-
-		var restriction deb.PackageQuery
-
-		restriction, err = changes.PackageQuery()
-		if err != nil {
-			failedFiles = append(failedFiles, path)
-			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
-			continue
-		}
-
-		var processedFiles2, failedFiles2 []string
-
-		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())
-
-		if err != nil {
-			return fmt.Errorf("unable to import package files: %s", err)
-		}
-
-		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
-
-		err = context.CollectionFactory().LocalRepoCollection().Update(repo)
-		if err != nil {
-			return fmt.Errorf("unable to save: %s", err)
-		}
-
-		err = changes.Cleanup()
-		if err != nil {
-			return err
-		}
-
-		for _, file := range failedFiles2 {
-			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		for _, file := range processedFiles2 {
-			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
-		}
-
-		processedFiles = append(processedFiles, path)
-	}
-
-	if !noRemoveFiles {
-		processedFiles = utils.StrSliceDeduplicate(processedFiles)
-
-		for _, file := range processedFiles {
-			err = os.Remove(file)
-			if err != nil {
-				return fmt.Errorf("unable to remove file: %s", err)
-			}
-		}
-	}
 
 	if len(failedFiles) > 0 {
 		context.Progress().ColoredPrintf("@y[!]@| @!Some files were skipped due to errors:@|")

cmd/repo_list.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/repo_move.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_remove.go

@@ -3,8 +3,8 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/repo_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/run.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	ctx "github.com/smira/aptly/context"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/smira/commander"
 )
 

cmd/serve.go

@@ -8,9 +8,9 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_create.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/snapshot_filter.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_list.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/snapshot_merge.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/snapshot_pull.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_rename.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/snapshot_search.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/query"
+	"github.com/aptly-dev/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_show.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )

cmd/snapshot_verify.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sort"
 
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 )
 

cmd/version.go

@@ -3,7 +3,7 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/smira/commander"
 )
 

completion.d/README-zsh.md

@@ -0,0 +1,65 @@
+# aptly-zsh
+zsh completion for aptly
+
+The zsh completion function and this README were imported from
+[steinymity's repository](https://github.com/steinymity/aptly-zsh).
+
+[Aptly](https://www.aptly.info/) is a great tool to setup Debian APT repositories
+and mirrors. However, it's sometimes hard to remember all the command line
+syntax and names of all options. Therefore I wrote this zsh completion modules
+for aptly.
+
+## License
+
+This project is licensed under the terms of the MIT license. See file `LICENSE`
+for details.
+
+## Installation
+
+Clone/copy the file `_aptly` to a place in your `$fpath` (show with
+`echo $fpath`), or create a new directory and extend the fpath:
+
+    mkdir -p ~/.zsh/functions
+    fpath=(~/.zsh/functions $fpath)
+    editor ~/.zsh/functions/_aptly
+
+To profit most from the provided help messages and completions, make sure that
+your zsh is setup properly. I have tested with the grml-zsh configuration that
+is [available on Github](https://github.com/grml/grml-etc-core/) and on the
+[grml homepage](http://grml.org/zsh/).
+
+## Compatibility
+
+The command line completion was developed based on the manpage of aptly 1.2.0
+(currently in Debian Testing). However, most completions will work on older
+versions (e.g., 0.9.7 in Debian Stable), too.
+
+The completion function completes most arguments and options that can be passed
+to aptly, including mirror/repository/snapshot/publish names. However, not all
+arguments are handled yet. See the next section for known limitations.
+
+## Known Bugs and Limitation
+
+ * Boolean options are always completed with an explicit value `true` or
+   `false`, although omitting the value is implicitly interpreted as `true`.
+ * The source and destination names of copy and move operations must not be the
+   same. This is currently not enforced.
+ * The package query and display format strings are currently not completed.
+ * Endpoints are not completed.
+ * In `publish snapshot` there is no connection between the number of
+   components passed to `-component` and the number of given snapshots to
+   publish. Furthermore, the help text for `endpoint:prefix` disappears
+   after its first possible location.
+ * In `publish switch` the distribution can be set independently of the
+   `endpoint:prefix` (i.e., all published distributions can be combined with
+   all published `endpoint:prefix`).
+ * Neither `publish switch` nor `publish update` check if publish was created
+   from a snapshot or directly from a local repo.
+ * Task commands are not completed. There are no commas added.
+ * `help` won't complete after a sub command (i.e., `aptly mirror list help`).
+ * Probably more, feel free to open issues, and submit patches/merge requests.
+
+--------------------------------------------------------------------------------
+
+Maximilian Stein <m@steiny.biz>
+2018-02-03

completion.d/_aptly

@@ -0,0 +1,617 @@
+#compdef aptly
+################################################################################
+# Copyright (C) 2018 Maximilian Stein <m@steiny.biz>
+#   Acknowledgement: Many texts where copied from aptly(1)
+#
+# This project is licensed under the terms of the MIT license.
+# See file LICENSE for details.
+################################################################################
+# zsh completion script for Aptly (http://aptly.info/)
+################################################################################
+
+# see https://man.cx/aptly
+
+local curcontext="$curcontext" state line cmd subcmd ret=1
+
+local arch_list=(amd64 arm64 armel armhf i386 mips mipsel mips64el ppc64el s390x)
+local bool="bool:{_wanted -V values expl 'bool' compadd true false}"
+local dists=({wheezy,jessie,stretch}{,-updates,-backports,-backports-sloopy} buster sid experimental)
+local components=(main contrib non-free)
+local aptly_query="aptly package query: "
+local aptly_format="aptly package display format: "
+local aptly_uploaders="-uploaders-file=[uploaders.json to be used when including .changes into this repository]:uploaders file:_files -g '*.json'"
+local keyring="*-keyring=[gpg keyring to use when verifying Release file (could be specified multiple times)]:keyring file:_files -g '*.gpg'"
+
+# complete command
+(( $+functions[_aptly-cmd] )) ||
+    _aptly-cmd() {
+        _values "aptly command category" \
+            "mirror[manage, update mirrors of remote repositories]" \
+            "repo[manage local package repositories, add, remove, move, copy packages]" \
+            "snapshot[create, merge, manage snapshots]" \
+            "package[perform operation on the whole collection of packages]" \
+            "publish[publish snapshot or local repository]" \
+            "db[cleanup database and package pool, recover database after failure]" \
+            "task[multi-command tasks]" \
+            "serve[quickly serve published repositories via HTTP]" \
+            "config[configuration management]" \
+            "graph[generate dependency graph]" \
+            "api[REST API service]"
+        ret=0
+}
+
+# complete subcommand
+(( $+functions[_aptly-subcmd] )) ||
+    _aptly-subcmd() {
+        local cmd=$1
+        case $cmd in
+            mirror)
+                _values "mirror commands" \
+                    "create[create new mirror of remote repository]" \
+                    "list[show full list of mirrors]" \
+                    "show[show details about a mirror]" \
+                    "drop[delete a mirror]" \
+                    "update[update a mirror]" \
+                    "rename[change name of a mirror]" \
+                    "edit[change settings of a mirror]" \
+                    "search[search mirror for packages matching query]"
+                ret=0 ;;
+            repo)
+                _values "repo commands" \
+                    "add[add packages to local repository]" \
+                    "copy[copy packages between local repositories]" \
+                    "create[create local repository]" \
+                    "drop[delete local repository]" \
+                    "edit[edit properties of local repository]" \
+                    "import[import packages from mirror to local repository]" \
+                    "list[list local repositories]" \
+                    "move[move packages between local repositories]" \
+                    "remove[remove packages from local repository]" \
+                    "show[show details about local repository]" \
+                    "rename[renames local repository]" \
+                    "search[search repo for packages matching query]" \
+                    "include[add packages to local repositories based on .changes files]"
+                ret=0 ;;
+            snapshot)
+                _values "snapshot commands" \
+                    "create[create snapshot of mirror or local repository]" \
+                    "list[list snapshots]" \
+                    "show[show details about snapshot]" \
+                    "verify[verify dependencies in snapshot]" \
+                    "pull[pull packages from another snapshot]" \
+                    "diff[show difference between two snapshots]" \
+                    "merge[merge snapshots]" \
+                    "drop[delete snapshot]" \
+                    "rename[rename snapshot]" \
+                    "search[search snapshot for packages matching query]" \
+                    "filter[filter packages in snapshot producing another snapshot]"
+                ret=0 ;;
+            publish)
+                _values "publish commands" \
+                    "drop[remove published repository]" \
+                    "list[list published repositories]" \
+                    "repo[publish local repository]" \
+                    "snapshot[publish snapshot]" \
+                    "switch[update published repository by switching to new snapshot]" \
+                    "update[update published local repository]" \
+                    "show[shows details of published repository]"
+                ret=0 ;;
+            package)
+                _values "package commands" \
+                    "search[search for packages matching query]" \
+                    "show[show details about packages matching query]"
+                ret=0 ;;
+            db)
+                _values "db commands" \
+                    "cleanup[cleanup db and package pool]" \
+                    "recover[recover db after crash]"
+                ret=0 ;;
+            serve)
+                # no subcommand here
+                _arguments '1:: :' \
+                    '-listen=[host:port for HTTP listening]:host\:port: '
+                ret=0 ;;
+            api)
+                _values "api commands" \
+                    "serve[start api http service]"
+                ret=0 ;;
+            graph)
+                # no subcommand here
+                _arguments '*:' \
+                    '-format=[render graph to specified format]:image format:(png svg pdf)' \
+                    '-layout=[create a more vertical or more horizontal graph layout]:layout:(horizontal vertical)' \
+                    '-output=[specify output filename, default is to open result in viewer]:output file:_files'
+                ret=0 ;;
+            config)
+                _values "config commands" \
+                    "show[show current aptly config]"
+                ret=0 ;;
+            task)
+                _values "task commands" \
+                    "run[run aptly tasks]"
+                ret=0 ;;
+        esac
+}
+
+# complete parameters
+(( $+functions[_aptly-param] )) ||
+    _aptly-param() {
+        local cmd=$1 subcmd=$2
+
+        local config=$opt_args[-config]
+        [[ -n $config ]] && config="-config=$config"
+
+        # get list of mirrors, or ' ' if none
+        get_mirrors() {
+            # retrieve list of mirrors
+            local mirrors=($(aptly $config mirror list -raw=true 2>/dev/null))
+            # a single space causes just the help text to be shown
+            [[ -z $mirrors ]] && mirrors=" " || mirrors="($mirrors)"
+            echo $mirrors
+        }
+        # get lists of repos or ' ' if none
+        get_repos() {
+            # retrieve repo list
+            local repos=($(aptly $config repo list -raw=true 2>/dev/null))
+            [[ -z $repos ]] && repos=" " || repos="($repos)"
+            echo $repos
+        }
+        # get list of snapshots or ' ' if none
+        get_snapshots() {
+            local snapshots=($(aptly $config snapshot list -raw=true 2>/dev/null))
+            [[ -z $snapshots ]] && snapshots=" " || snapshots="($snapshots)"
+            echo $snapshots
+        }
+        # get list of gpg keys or ' ' if none
+        get_gpg_key_ids() {
+            local gpg_keys=($(gpg --quiet --batch  --keyid-format long --list-secret-keys --with-colons 2>/dev/null | grep '^sec' | cut -d ':' -f 5))
+            [[ -z $gpg_keys ]] && gpg_keys=" " || gpg_keys="($gpg_keys)"
+            echo $gpg_keys
+        }
+
+        ret=0
+        case $cmd in
+            mirror)
+                local mirrors=$(get_mirrors)
+
+                case $subcmd in
+                    create)
+                        _arguments \
+                            "-filter=[filter packages in mirror]:$aptly_query" \
+                            "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
+                            "-force-architecture=[(only with architecture list) skip check that requested architectures are listed in Release file]:$bool" \
+                            "-force-components=[(only with component list) skip check that requested components are listed in Release file]:$bool" \
+                            "-ignore-signatures=[disable verification of Release file signatures]:$bool" \
+                            $keyring \
+                            "-with-sources=[download source packages in addition to binary packages]:$bool" \
+                            "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine-readable format]:$bool"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show detailed list of packages and versions stored in the mirror]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[force mirror deletion even if used by snapshots]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    update)
+                        _arguments \
+                            "-download-limit=[limit download speed (kB/s)]:kB/s: " \
+                            "-force=[force update mirror even if it is locked by another process]:$bool" \
+                            "-ignore-checksums=[ignore checksum mismatches while downloading package files and metadata]:$bool" \
+                            "-ignore-signatures=[disable verification of Release file signatures]:$bool" \
+                            $keyring \
+                            "-max-tries=[max download tries till process fails with download error]:number: " \
+                            "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    rename)
+                        _arguments \
+                            "2:old mirror name:$mirrors" ":new mirror name: "
+                        ;;
+                    edit)
+                        _arguments \
+                            "-filter=[filter packages in mirror]:$aptly_query" \
+                            "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
+                            "-with-sources=[download source packages in addition to binary packages]:$bool" \
+                            "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "(-)2:mirror name:$mirrors"
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:mirror name:$mirrors" ":$aptly_query"
+                        ;;
+                esac
+                ;;
+
+            repo)
+
+
+                local repos=$(get_repos)
+
+                local create_edit=("-comment=[any text that would be used to described local repository]:comment: "
+                            "-component=[default component when publishing]:component:($components)"
+                            "-distribution=[default distribution when publishing]:distribution:($dists)"
+                            $aptly_uploaders
+                            )
+
+                case $subcmd in
+                    add)
+                        _arguments \
+                            "-force-replace=[when adding package that conflicts with existing package, remove existing package]:$bool" \
+                            "-remove-files=[remove files that have been imported successfully into repository]:$bool" \
+                            "(-)2:repo name:$repos" "*:package files:_files -g '*.{udeb,deb,dsc}'"
+                        ;;
+                    copy)
+                        _arguments \
+                            "-dry-run=[don’t copy, just show what would be copied]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$$bool" \
+                            "(-)2:src repo name:$repos" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    create)
+                        local snapshots=$(get_snapshots)
+
+                        _arguments \
+                            ${create_edit[@]} \
+                            "(-)2:new repo name: " \
+                            "3:::('from')" "4:::('snapshot')" "5::snapshot:$snapshots"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[force local repo deletion even if used by snapshots]:$bool" \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    edit)
+                        _arguments \
+                            ${create_edit[@]} \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    import)
+                        local mirrors=$(get_mirrors)
+
+                        _arguments \
+                            "-dry-run=[don’t import, just show what would be imported]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$bool" \
+                            "(-)2:src mirror name:$mirrors" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine−readable format]:$bool"
+                        ;;
+                    move)
+                        _arguments \
+                            "-dry-run=[don’t move, just show what would be moved]:$bool" \
+                            "-with-deps=[follow dependencies when processing package−spec]:$bool" \
+                            "(-)2:srv repo name:$repos" ":dest repo name:$repos" "*:$aptly_query"
+                        ;;
+                    remove)
+                        _arguments \
+                            "-dry-run=[don’t remove, just show what would be removed]:$bool" \
+                            "(-)2:repo name:$repos" "*:$aptly_query"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show list of packages]:$bool" \
+                            "(-)2:repo name:$repos"
+                        ;;
+                    rename)
+                        _arguments \
+                            "2:old repo name:$repos" ":new repo name: "
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:repo name:$repos" ":$aptly_query"
+                        ;;
+                    include)
+                        _arguments '1:: :' \
+                            "-accept-unsigned=[accept unsigned .changes files]:$bool" \
+                            "-force-replace=[when adding package that conflicts with existing package, remove existing package]:$bool" \
+                            "-ignore-signatures=[disable verification of .changes file signature]:$bool" \
+                            $keyring \
+                            "-no-remove-files=[don’t remove files that have been imported successfully into repository]:$bool" \
+                            "-repo=[which repo should files go to, defaults to Distribution field of .changes file]:repo name:$repos" \
+                            $aptly_uploaders \
+                            "(-)*:changes files/directories:_files -g '*.changes'"
+                        ;;
+                esac
+                ;;
+            snapshot)
+                local snapshots=$(get_snapshots)
+
+                case $subcmd in
+                    create)
+                        local mirrors=$(get_mirrors)
+                        local repos=$(get_repos)
+
+                        _arguments -C \
+                            '(-)2:new snapshot name: ' \
+                            '3: :->src1' \
+                            '4:: :->src2' '5:: :->src3'
+
+                        case $state in
+                            src1)
+                                _values 'snapshot src' 'from' 'empty' ;;
+                            src2)
+                                if [[ $line[3] == from ]]; then
+                                    _values 'snapshot src' 'mirror' 'repo'
+                                fi
+                                ;;
+                            src3)
+                                if [[ $line[3] == from ]]; then
+                                    case $line[4] in
+                                        mirror)
+                                            _arguments "5:mirror name:$mirrors" ;;
+                                        repo)
+                                            _arguments "5:repo name:$repos" ;;
+                                    esac
+                                fi
+                                ;;
+                        esac
+                        ;;
+                    list)
+                        _arguments '1:: :' \
+                            "-raw=[display list in machine−readable format]:$bool" \
+                            "-sort=[display list in ’name’ or creation ’time’ order]:sort order:((name\:'alphabetical order' time\:'chronological order'))"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-packages=[show list of packages]:$bool" \
+                            "(-)2:snapshot name:$snapshots"
+                        ;;
+                    verify)
+                        _arguments '1:: :' \
+                            "(-)2:snapshot name:$snapshots" "*::more snapshots:$snapshots"
+                        ;;
+                    pull)
+                        _arguments \
+                            "-all-matches=[pull all the packages that satisfy the dependency version requirements]:$bool" \
+                            "-dry-run=[don’t create destination snapshot, just show what would be pulled]:$bool" \
+                            "-no-deps=[don’t process dependencies, just pull listed packages]:$bool" \
+                            "-no-remove=[don’t remove other package versions when pulling package]:$bool" \
+                            "(-)2:to snapshot name:$snapshots" "3:src snapshot name:$snapshots" "4:new dest snapshot name: " \
+                            "*:$aptly_query"
+                        ;;
+                    diff)
+                        _arguments \
+                            "-only-matching=[display diff only for matching packages (don’t display missing packages)]:$bool" \
+                            "(-)2:snapshot name a:$snapshots" "3:snapshot name b:$snapshots"
+                        ;;
+                    merge)
+                        _arguments \
+                            "-latest=[use only the latest version of each package]:$bool" \
+                            "-no-remove=[don’t remove duplicate arch/name packages]:$bool" \
+                            "(-)2:new dest snapshot name: " "*:source snapshot name(s):$snapshots"
+                        ;;
+                    drop)
+                        _arguments \
+                            "-force=[remove snapshot even if it was used as source for other snapshots]:$bool" \
+                            "(-)2:snapshot name:$snapshots"
+                        ;;
+                    rename)
+                        _arguments '1:: :' \
+                            "2:old snapshot name:$snapshots" "3:new snapshot name: "
+                        ;;
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "-with-deps=[include dependencies into search results]:$bool" \
+                            "(-)2:snapshot name:$snapshots" ":$aptly_query"
+                        ;;
+                    filter)
+                        _arguments \
+                            "-with-deps=[include dependent packages as well]:$bool" \
+                            "(-)2:src snapshot name:$snapshots" "3:new dest snapshot name: " "*:$aptly_query"
+                        ;;
+                esac
+                ;;
+            publish)
+                # read lines of output into
+                # format of each item: <ep:prefix> <distribution>
+                local -a publish_prefixes
+                local -a publish_dists
+                for line in ${(@f)"$(aptly $config publish list -raw=true 2>/dev/null)"}
+                {
+                    publish_prefixes+=($line[(ws: :)1])
+                    publish_dists+=($line[(ws: :)2])
+                }
+                publish_prefixes_uniq=(${(@u)publish_prefixes})
+                publish_dists_uniq=(${(@u)publish_dists})
+                [[ -z $publish_prefixes_uniq ]] && publish_prefixes_uniq=" " || publish_prefixes_uniq="($publish_prefixes_uniq)"
+                [[ -z $publish_dists_uniq ]]    && publish_dists_uniq=" "    || publish_dists_uniq="($publish_dists_uniq)"
+
+                local gpg_keys=$(get_gpg_key_ids)
+
+                # common options for publishing
+                # TODO: is the keyring parameter correct?
+                local publish_update_options=(
+                            "-batch=[run GPG with detached tty]:$bool"
+                            "-force-overwrite=[overwrite files in package pool in case of mismatch]:$bool"
+                            "-gpg-key=[GPG key ID to use when signing the release]:gpg key id:$gpg_keys"
+                            "-keyring=[GPG keyring to use (instead of default)]:keyring file:_files -g '*.gpg'"
+                            "-passphrase=[GPG passhprase for the key (warning: could be insecure)]:passphrase: "
+                            "-passphrase-file=[GPG passhprase−file for the key (warning: could be insecure)]:passphrase file:_files"
+                            "-secret-keyring=[GPG secret keyring to use (instead of default)]:secret-keyring:_files"
+                            "-skip-contents=[don’t generate Contents indexes]:$bool"
+                            "-skip-signing=[don’t sign Release files with GPG]:$bool"
+                )
+                local components_options=(
+                            "-component=[component name to publish (for multi−component publishing, separate components with commas)]:components:_values -s , components $components"
+                )
+                local publish_options=(
+                            "-butautomaticupgrades=[set value for ButAutomaticUpgrades field]:$bool"
+                            "-distribution=[distribution name to publish]:distribution:($dists)"
+                            "-label=[label to publish]:label: "
+                            "-notautomatic=[set value for NotAutomatic field]:notautomatic: "
+                            "-origin=[origin name to publish]:origin: "
+                            ${components_options[@]}
+                )
+
+                local endpoint_prefix="[endpoint\:]prefix"
+
+                case $subcmd in
+                    repo)
+                        local repos=$(get_repos)
+                        _arguments \
+                            ${publish_options[@]} \
+                            ${publish_update_options[@]} \
+                            "(-)2:repo name:$repos" "3::$endpoint_prefix: "
+                        ;;
+                    snapshot)
+                        local snapshots=$(get_snapshots)
+                        _arguments '1:: :' \
+                            ${publish_options[@]} \
+                            ${publish_update_options[@]} \
+                            "(-)*:snapshot name:$snapshots" "3::$endpoint_prefix: "
+                        ;;
+                    switch)
+                        local snapshots=$(get_snapshots)
+                        _arguments \
+                            ${publish_update_options[@]} \
+                            ${components_options[@]} \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq" \
+                            "*:new snapshot name:$snapshots"
+                        ;;
+                    update)
+                        _arguments \
+                            ${publish_update_options[@]} \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq"
+                        ;;
+                    show)
+                        _arguments '1:: :' \
+                            "(-)2:distribution:$publish_dists_uniq" "3::$endpoint_prefix:$publish_prefixes_uniq"
+                        ;;
+                esac
+                ;;
+            package)
+                case $subcmd in
+                    search)
+                        _arguments \
+                            "-format=[custom format for result printing]:$aptly_format" \
+                            "(-)2:$aptly_query"
+                        ;;
+                    show)
+                        _arguments \
+                            "-with-files=[display information about files from package pool]:$bool" \
+                            "-with-references=[display information about mirrors, snapshots and local repos referencing this package]:$bool" \
+                            "(-)2:$aptly_query"
+                        ;;
+                esac
+                ;;
+            db)
+                case $subcmd in
+                    cleanup)
+                        _arguments '1:: :' \
+                            "-dry-run=[don’t delete anything]:$bool" \
+                            "-verbose=[be verbose when loading objects/removing them]:$bool"
+                        ;;
+                    recover)
+                        # nothing to complete...
+                        ;;
+                esac
+                ;;
+            serve)
+                # completed in _aptly-subcmd
+                ;;
+            api)
+                case $subcmd in
+                    serve)
+                        _arguments '1:: :' \
+                            "-listen=[host:port for HTTP listening or unix://path to listen on a Unix domain socket]:host\:port or unix\://path: " \
+                            "-no-lock=[don’t lock the database]:$bool"
+                        ;;
+                esac
+                ;;
+            graph)
+                # completed in _aptly-subcmd
+                ;;
+            config)
+                case $subcmd in
+                    show)
+                        # nothing to do
+                        ;;
+                esac
+                ;;
+            task)
+                case $subcmd in
+                    run)
+                        _arguments '1:: :' \
+                            "(2)-filename=[specifies the filename that contains the commands to run]:filename:_files" \
+                            "(-filename)*::comma-separated command list: "
+                esac
+                ;;
+        esac
+}
+
+# main completion
+_arguments -C \
+    "-architectures=[list of architectures to consider (comma−separated), default to all available]:architectures:_values -s , architectures $arch_list" \
+    "-config=[location of configuration file]:file:_files -g '*.conf'" \
+    "-db-open-attempts=[number of attempts to open DB if it’s locked by other instance]:number:()" \
+    "-dep-follow-all-variants=[when processing dependencies, follow a & b if dependency is ’a|b’]:$bool" \
+    "-dep-follow-recommends=[when processing dependencies, follow Recommends]:$bool" \
+    "-dep-follow-source=[when processing dependencies, follow from binary to Source packages]:$bool" \
+    "-dep-follow-suggests=[when processing dependencies, follow Suggests]:$bool" \
+    "-dep-verbose-resolve=[when processing dependencies, print detailed logs]:$bool" \
+    "-gpg-provider=[PGP implementation]:gpg provider:((gpg\:'external gpg' internal\:'Go internal implementation'))" \
+    '(-)1: :->cmds' \
+    '2: :->subcmd' \
+    '*:: :->args' && ret=0
+
+cmd=$line[1]
+subcmd=$line[2]
+
+case $state in
+    cmds)
+        _aptly-cmd
+        _arguments '(-)1:: :((help\:integrated\ command\ help))'
+        ;;
+    subcmd)
+        case $cmd in
+            help)
+                _aptly-cmd
+                ;;
+            *)
+                _aptly-subcmd $cmd
+                _arguments '(-)2:: :((help\:integrated\ command\ help))'
+                ;;
+        esac
+        ;;
+
+    args)
+        # help anywhere in line?
+        if [[ ${line[(i)help]} -le ${#line} ]]; then
+            if [[ ${#line} -le 3 ]]; then
+                if [[ $line[1] == help ]]; then
+                    _aptly-subcmd $subcmd
+                elif [[ $line[2] == help ]]; then
+                    _aptly-subcmd $cmd
+                fi
+            fi
+        else
+            _aptly-param $cmd $subcmd
+            # this somehow destroys parameter completion, so disable it for now
+            #_arguments '(-)3:: :((help\:integrated\ command\ help))'
+        fi
+        ret=0
+        ;;
+esac
+
+return ret
+
+# mode: Shell-Script
+# sh-indentation: 4
+# indent-tabs-mode: nil
+# sh-basic-offset: 4
+# End:
+# vim: ft=zsh sw=4 ts=4 et

completion.d/aptly

@@ -156,7 +156,7 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
@@ -164,7 +164,7 @@ _aptly()
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi
@@ -499,7 +499,7 @@ _aptly()
           "snapshot"|"repo")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
               else
                 if [[ "$subcmd" == "snapshot" ]]; then
                   COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
@@ -524,7 +524,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -539,7 +539,7 @@ _aptly()
           "switch")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi
@@ -559,7 +559,7 @@ _aptly()
           "drop")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force-drop" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
               fi

console/progress.go

@@ -5,8 +5,8 @@ import (
 	"os"
 	"strings"
 
+	"github.com/aptly-dev/aptly/aptly"
 	"github.com/cheggaaa/pb"
-	"github.com/smira/aptly/aptly"
 	"github.com/wsxiaoys/terminal/color"
 )
 

context/context.go

@@ -2,9 +2,11 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
 	"math/rand"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
@@ -12,16 +14,16 @@ import (
 	"sync"
 	"time"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/console"
+	"github.com/aptly-dev/aptly/console"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/deb"
+	"github.com/aptly-dev/aptly/deb"
-	"github.com/smira/aptly/files"
+	"github.com/aptly-dev/aptly/files"
-	"github.com/smira/aptly/http"
+	"github.com/aptly-dev/aptly/http"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/s3"
+	"github.com/aptly-dev/aptly/s3"
-	"github.com/smira/aptly/swift"
+	"github.com/aptly-dev/aptly/swift"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -30,6 +32,8 @@ import (
 type AptlyContext struct {
 	sync.Mutex
 
+	gocontext.Context
+
 	flags, globalFlags *flag.FlagSet
 	configLoaded       bool
 
@@ -329,7 +333,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 		} else if strings.HasPrefix(name, "filesystem:") {
 			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
-				Fatal(fmt.Errorf("published local storage %v not configured", name[6:]))
+				Fatal(fmt.Errorf("published local storage %v not configured", name[11:]))
 			}
 
 			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
@@ -383,23 +387,42 @@ func (context *AptlyContext) pgpProvider() string {
 		provider = context.config().GpgProvider
 	}
 
-	if !(provider == "gpg" || provider == "internal") { // nolint: goconst
+	switch provider {
+	case "gpg": // nolint: goconst
+	case "gpg1": // nolint: goconst
+	case "gpg2": // nolint: goconst
+	case "internal": // nolint: goconst
+	default:
 		Fatal(fmt.Errorf("unknown gpg provider: %v", provider))
 	}
 
 	return provider
 }
 
+func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
+	switch context.pgpProvider() {
+	case "gpg1":
+		return pgp.GPG1Finder()
+	case "gpg2":
+		return pgp.GPG2Finder()
+	case "gpg":
+		return pgp.GPGDefaultFinder()
+	}
+
+	panic("uknown GPG provider type")
+}
+
 // GetSigner returns Signer with respect to provider
 func (context *AptlyContext) GetSigner() pgp.Signer {
 	context.Lock()
 	defer context.Unlock()
 
-	if context.pgpProvider() == "gpg" { // nolint: goconst
+	provider := context.pgpProvider()
-		return &pgp.GpgSigner{}
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoSigner{}
 	}
 
-	return &pgp.GoSigner{}
+	return pgp.NewGpgSigner(context.getGPGFinder(provider))
 }
 
 // GetVerifier returns Verifier with respect to provider
@@ -407,11 +430,12 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
 	context.Lock()
 	defer context.Unlock()
 
-	if context.pgpProvider() == "gpg" { // nolint: goconst
+	provider := context.pgpProvider()
-		return &pgp.GpgVerifier{}
+	if provider == "internal" { // nolint: goconst
+		return &pgp.GoVerifier{}
 	}
 
-	return &pgp.GoVerifier{}
+	return pgp.NewGpgVerifier(context.getGPGFinder(provider))
 }
 
 // UpdateFlags sets internal copy of flags in the context
@@ -438,6 +462,27 @@ func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
 	return context.globalFlags
 }
 
+// GoContextHandleSignals upgrades context to handle ^C by aborting context
+func (context *AptlyContext) GoContextHandleSignals() {
+	context.Lock()
+	defer context.Unlock()
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	var cancel gocontext.CancelFunc
+
+	context.Context, cancel = gocontext.WithCancel(context.Context)
+
+	go func() {
+		<-sigch
+		signal.Stop(sigch)
+		context.Progress().PrintfStdErr("Aborting... press ^C once again to abort immediately\n")
+		cancel()
+	}()
+}
+
 // Shutdown shuts context down
 func (context *AptlyContext) Shutdown() {
 	context.Lock()
@@ -494,6 +539,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 		flags:             flags,
 		globalFlags:       flags,
 		dependencyOptions: -1,
+		Context:           gocontext.TODO(),
 		publishedStorages: map[string]aptly.PublishedStorage{},
 	}
 

context/context_test.go

@@ -0,0 +1,86 @@
+package context
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type fatalErrorPanicChecker struct {
+	*CheckerInfo
+}
+
+var FatalErrorPanicMatches Checker = &fatalErrorPanicChecker{
+	&CheckerInfo{Name: "FatalErrorPanics", Params: []string{"function", "expected"}},
+}
+
+func (checker *fatalErrorPanicChecker) Check(params []interface{}, names []string) (result bool, errmsg string) {
+	f := reflect.ValueOf(params[0])
+	if f.Kind() != reflect.Func || f.Type().NumIn() != 0 {
+		return false, "Function must take zero arguments"
+	}
+	defer func() {
+		if errmsg != "" {
+			return
+		}
+
+		obtained := recover()
+
+		names[0] = "panic"
+
+		var ok bool
+
+		var e1 *FatalError
+		if e1, ok = obtained.(*FatalError); ok {
+			params[0] = e1
+		} else {
+			errmsg = "Panic value is not FatalError"
+			return
+		}
+
+		var e2 *FatalError
+		if e2, ok = params[1].(*FatalError); ok {
+			params[1] = e2
+		} else {
+			errmsg = "Expected value is not FatalError"
+			return
+		}
+
+		if *e1 == *e2 {
+			result = true
+		} else {
+			result = false
+			errmsg = "Not equal"
+		}
+	}()
+	f.Call(nil)
+	return false, "Function has not panicked"
+}
+
+type AptlyContextSuite struct {
+	context *AptlyContext
+}
+
+var _ = Suite(&AptlyContextSuite{})
+
+func (s *AptlyContextSuite) SetUpTest(c *C) {
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.String("config", "", "")
+	context, err := NewContext(flags)
+	c.Assert(err, IsNil)
+	s.context = context
+}
+
+func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
+	// https://github.com/aptly-dev/aptly/issues/711
+	// This will fail on account of us not having a config, so the
+	// storage never exists.
+	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
+		FatalErrorPanicMatches,
+		&FatalError{ReturnCode: 1, Message: "published local storage fuji not configured"})
+}

deb/changes.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -8,10 +9,11 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"text/template"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // Changes is a result of .changes file parsing
@@ -93,8 +95,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		text = input
 	}
 
-	reader := NewControlFileReader(text)
+	reader := NewControlFileReader(text, false, false)
-	c.Stanza, err = reader.ReadStanza(false)
+	c.Stanza, err = reader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -164,7 +166,7 @@ func (c *Changes) Cleanup() error {
 }
 
 // PackageQuery returns query that every package should match to be included
-func (c *Changes) PackageQuery() (PackageQuery, error) {
+func (c *Changes) PackageQuery() PackageQuery {
 	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
 	for _, arch := range c.Architectures {
 		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
@@ -215,7 +217,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
 	}
 
-	return &AndQuery{L: archQuery, R: nameQuery}, nil
+	return &AndQuery{L: archQuery, R: nameQuery}
 }
 
 // GetField implements PackageLike interface
@@ -288,3 +290,143 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 
 	return
 }
+
+// ImportChangesFiles imports referenced files in changes files into local repository
+func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
+	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
+	}
+	for _, path := range changesFiles {
+		var changes *Changes
+
+		changes, err = NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		if progress != nil {
+			progress.Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+		}
+
+		var repo *LocalRepo
+		repo, err = localRepoCollection.ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = parseQuery(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return nil, nil, fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		var list *PackageList
+		list, err = NewPackageListFromRefList(repo.RefList(), packageCollection, progress)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, otherFiles, _ := CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		restriction := changes.PackageQuery()
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
+			packageCollection, reporter, restriction, checksumStorage)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
+
+		err = localRepoCollection.Update(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err = os.Remove(file)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	return processedFiles, failedFiles, nil
+}

deb/changes_test.go

@@ -4,24 +4,52 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type ChangesSuite struct {
-	Dir, Path string
+	Dir, Path           string
+	Reporter            aptly.ResultReporter
+	db                  database.Storage
+	localRepoCollection *LocalRepoCollection
+	packageCollection   *PackageCollection
+	packagePool         aptly.PackagePool
+	checksumStorage     aptly.ChecksumStorage
+	progress            aptly.Progress
 }
 
 var _ = Suite(&ChangesSuite{})
 
 func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Reporter = &aptly.RecordingResultReporter{
+		Warnings:     []string{},
+		AddedLines:   []string{},
+		RemovedLines: []string{},
+	}
 	s.Dir = c.MkDir()
 	s.Path = filepath.Join(s.Dir, "calamares.changes")
-
+	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
-	f, err := os.Create(s.Path)
 	c.Assert(err, IsNil)
 
-	f.WriteString(changesFile)
+	s.db, _ = database.NewOpenDB(c.MkDir())
-	f.Close()
+	s.localRepoCollection = NewLocalRepoCollection(s.db)
+	s.packageCollection = NewPackageCollection(s.db)
+
+	s.checksumStorage = files.NewMockChecksumStorage()
+	s.packagePool = files.NewPackagePool(s.Dir, false)
+	s.progress = console.NewProgress()
+	s.progress.Start()
+}
+
+func (s *ChangesSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
+	s.db.Close()
 }
 
 func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -44,6 +72,73 @@ func (s *ChangesSuite) TestParseAndVerify(c *C) {
 	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
 }
 
+func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
+	changesFiles, failedFiles := CollectChangesFiles([]string{"testdata/changes"}, s.Reporter)
+
+	c.Check(failedFiles, HasLen, 0)
+	c.Check(changesFiles, DeepEquals, []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	})
+}
+
+func (s *ChangesSuite) TestImportChangesFiles(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	origFailedFiles := []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.0_i386.deb",
+	}
+	origProcessedFiles := []string{
+		"testdata/changes/hardlink_0.2.1.dsc",
+		"testdata/changes/hardlink_0.2.1.tar.gz",
+		"testdata/changes/hardlink_0.2.1_amd64.deb",
+		"testdata/changes/hardlink_0.2.1_amd64.buildinfo",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	}
+
+	var expectedProcessedFiles, expectedFailedFiles []string
+
+	for _, path := range origFailedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedFailedFiles = append(expectedFailedFiles, filename)
+	}
+
+	for _, path := range origProcessedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedProcessedFiles = append(expectedProcessedFiles, filename)
+	}
+
+	changesFiles, failedFiles := CollectChangesFiles([]string{s.Dir}, s.Reporter)
+	c.Check(failedFiles, HasLen, 0)
+
+	processedFiles, failedFiles, err := ImportChangesFiles(
+		append(changesFiles, "testdata/changes/notexistent.changes"),
+		s.Reporter, true, true, false, false, &NullVerifier{},
+		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
+	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
+}
+
+func (s *ChangesSuite) TestPrepare(c *C) {
+	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
+	c.Assert(err, IsNil)
+	err = changes.Prepare()
+	c.Assert(err, IsNil)
+
+	_, err = os.Stat(filepath.Join(changes.TempDir, "hardlink_0.2.1_amd64.changes"))
+	c.Check(err, IsNil)
+}
+
 func (s *ChangesSuite) TestPackageQuery(c *C) {
 	changes, err := NewChanges(s.Path)
 	c.Assert(err, IsNil)
@@ -51,42 +146,7 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 	err = changes.VerifyAndParse(true, true, &NullVerifier{})
 	c.Check(err, IsNil)
 
-	q, err := changes.PackageQuery()
+	q := changes.PackageQuery()
-	c.Check(err, IsNil)
-
 	c.Check(q.String(), Equals,
 		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
 }
-
-var changesFile = `Format: 1.8
-Date: Thu, 27 Nov 2014 13:24:53 +0000
-Source: calamares
-Binary: calamares calamares-dbg
-Architecture: source amd64
-Version: 0+git20141127.99
-Distribution: sid
-Urgency: medium
-Maintainer: Rohan Garg <rohan@kde.org>
-Changed-By: Rohan <rohan@kde.org>
-Description:
- calamares  - distribution-independent installer framework
- calamares-dbg - distribution-independent installer framework -- debug symbols
-Changes:
- calamares (0+git20141127.99) sid; urgency=medium
- .
-   * Update from git
-Checksums-Sha1:
- 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
- 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
- d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
- a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Checksums-Sha256:
- 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
- 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
- 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
- 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Files:
- 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
- 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
- 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
- e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/checksum_collection.go

@@ -3,9 +3,9 @@ package deb
 import (
 	"bytes"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/ugorji/go/codec"
 )
 

deb/checksum_collection_test.go

@@ -1,8 +1,8 @@
 package deb
 
 import (
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )

deb/collections.go

@@ -3,7 +3,7 @@ package deb
 import (
 	"sync"
 
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 )
 
 // CollectionFactory is a single place to generate all desired collections

deb/contents.go

@@ -6,9 +6,8 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/database"
+	"github.com/pborman/uuid"
-	"github.com/smira/go-uuid/uuid"
 )
 
 // ContentsIndex calculates mapping from files to packages, with sorting and aggregation
@@ -26,10 +25,7 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
+func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
-	contents := p.Contents(packagePool, progress)
-	qualifiedName := []byte(p.QualifiedName())
-
 	for _, path := range contents {
 		// for performance reasons we only write to leveldb during push.
 		// merging of qualified names per path will be done in WriteTo

deb/deb.go

@@ -11,12 +11,12 @@ import (
 	"strings"
 
 	"github.com/h2non/filetype/matchers"
-	"github.com/mkrautz/goar"
+	ar "github.com/mkrautz/goar"
 	"github.com/pkg/errors"
 
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
+	"github.com/kjk/lzma"
 	"github.com/smira/go-xz"
-	"github.com/smira/lzma"
 )
 
 // Source kinds
@@ -26,6 +26,8 @@ const (
 	SourceRemoteRepo = "repo"
 )
 
+type parseQuery func(string) (PackageQuery, error)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -37,21 +39,46 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
+
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
+			return nil, fmt.Errorf("unable to find control.tar.* part in package %s", packageFile)
 		}
 		if err != nil {
 			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
-		if header.Name == "control.tar.gz" {
+		// As per deb(5) version 1.19.0.4 the control file may be:
-			ungzip, err := gzip.NewReader(library)
+		// - control.tar (since 1.17.6)
-			if err != nil {
+		// - control.tar.gz
-				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
+		// - control.tar.xz (since 1.17.6)
-			}
+		// Look for all of the above and uncompress as necessary.
-			defer ungzip.Close()
+		if strings.HasPrefix(header.Name, "control.tar") {
+			bufReader := bufio.NewReader(library)
 
-			untar := tar.NewReader(ungzip)
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "control.tar":
+				tarInput = bufReader
+			case "control.tar.gz":
+				ungzip, err := gzip.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "control.tar.xz":
+				unxz, err := xz.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
@@ -62,8 +89,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
-					reader := NewControlFileReader(untar)
+					reader := NewControlFileReader(untar, false, false)
-					stanza, err := reader.ReadStanza(false)
+					stanza, err := reader.ReadStanza()
 					if err != nil {
 						return nil, err
 					}
@@ -102,8 +129,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		text = file
 	}
 
-	reader := NewControlFileReader(text)
+	reader := NewControlFileReader(text, false, false)
-	stanza, err := reader.ReadStanza(false)
+	stanza, err := reader.ReadStanza()
 	if err != nil {
 		return nil, err
 	}

deb/deb_test.go

@@ -5,13 +5,13 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
 
 	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {
-	debFile, debFile2, dscFile, dscFileNoSign string
+	debFile, debFile2, debFileWithXzControl, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -20,6 +20,7 @@ func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
 	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
+	s.debFileWithXzControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.9.1+dfsg-2+18.04+bionic+build4_amd64.ddeb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -38,6 +39,14 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 	c.Check(st["Package"], Equals, "libboost-program-options-dev")
 }
 
+func (s *DebSuite) TestGetControlFileFromDebWithXzControl(c *C) {
+	// Has control.tar.xz archive inside.
+	st, err := GetControlFileFromDeb(s.debFileWithXzControl)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "5.9.1+dfsg-2+18.04+bionic+build4")
+	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
+}
+
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 	verifier := &pgp.GoVerifier{}
 

deb/format.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"io"
+	"sort"
 	"strings"
 	"unicode"
 )
@@ -11,6 +12,9 @@ import (
 // Stanza or paragraph of Debian control file
 type Stanza map[string]string
 
+// MaxFieldSize is maximum stanza field size in bytes
+const MaxFieldSize = 2 * 1024 * 1024
+
 // Canonical order of fields in stanza
 // Taken from: http://bazaar.launchpad.net/~ubuntu-branches/ubuntu/vivid/apt/vivid/view/head:/apt-pkg/tagfile.cc#L504
 var (
@@ -85,6 +89,9 @@ var (
 		"Directory",
 		"Files",
 	}
+	canonicalOrderInstaller = []string{
+		"",
+	}
 )
 
 // Copy returns copy of Stanza
@@ -98,6 +105,9 @@ func (s Stanza) Copy() (result Stanza) {
 
 func isMultilineField(field string, isRelease bool) bool {
 	switch field {
+	// file without a section
+	case "":
+		return true
 	case "Description":
 		return true
 	case "Files":
@@ -124,26 +134,33 @@ func isMultilineField(field string, isRelease bool) bool {
 	return false
 }
 
-// Write single field from Stanza to writer
+// Write single field from Stanza to writer.
+//
+//nolint: interfacer
 func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
 	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
-		if !strings.HasSuffix(value, "\n") {
+		if field != "" && !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
 
-		if field != "Description" {
+		if field != "Description" && field != "" {
 			value = "\n" + value
 		}
-		_, err = w.WriteString(field + ":" + value)
+
+		if field != "" {
+			_, err = w.WriteString(field + ":" + value)
+		} else {
+			_, err = w.WriteString(value)
+		}
 	}
 
 	return
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
 	canonicalOrder := canonicalOrderBinary
 	if isSource {
 		canonicalOrder = canonicalOrderSource
@@ -151,6 +168,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	if isRelease {
 		canonicalOrder = canonicalOrderRelease
 	}
+	if isInstaller {
+		canonicalOrder = canonicalOrderInstaller
+	}
 
 	for _, field := range canonicalOrder {
 		value, ok := s[field]
@@ -163,10 +183,21 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		}
 	}
 
-	for field, value := range s {
+	// no extra fields in installer
-		err := writeField(w, field, value, isRelease)
+	if !isInstaller {
-		if err != nil {
+		// Print extra fields in deterministic order (alphabetical)
-			return err
+		keys := make([]string, len(s))
+		i := 0
+		for field := range s {
+			keys[i] = field
+			i++
+		}
+		sort.Strings(keys)
+		for _, field := range keys {
+			err := writeField(w, field, s[field], isRelease)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -209,19 +240,28 @@ func canonicalCase(field string) string {
 
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
-	scanner *bufio.Scanner
+	scanner     *bufio.Scanner
+	isRelease   bool
+	isInstaller bool
 }
 
 // NewControlFileReader creates ControlFileReader, it wraps with buffering
-func NewControlFileReader(r io.Reader) *ControlFileReader {
+func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
-	return &ControlFileReader{scanner: bufio.NewScanner(bufio.NewReaderSize(r, 32768))}
+	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
+	scnr.Buffer(nil, MaxFieldSize)
+
+	return &ControlFileReader{
+		scanner:     scnr,
+		isRelease:   isRelease,
+		isInstaller: isInstaller,
+	}
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
+func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
-	lastFieldMultiline := false
+	lastFieldMultiline := c.isInstaller
 
 	for c.scanner.Scan() {
 		line := c.scanner.Text()
@@ -234,7 +274,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 			continue
 		}
 
-		if line[0] == ' ' || line[0] == '\t' {
+		if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
@@ -246,7 +286,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			lastFieldMultiline = isMultilineField(lastField, isRelease)
+			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bufio"
 	"bytes"
+	"os"
 	"strings"
 
 	. "gopkg.in/check.v1"
@@ -14,6 +15,10 @@ type ControlFileSuite struct {
 
 var _ = Suite(&ControlFileSuite{})
 
+const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list
+9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+`
+
 const controlFile = `Package: bti
 Binary: bti
 Version: 032-1
@@ -82,15 +87,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
 
-	stanza1, err := r.ReadStanza(false)
+	stanza1, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza(false)
+	stanza3, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
@@ -102,27 +107,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
-	stanza, err := r.ReadStanza(false)
+	stanza, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, true, false)
+	err = stanza.Copy().WriteTo(w, true, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
 
 	str := buf.String()
 
-	r = NewControlFileReader(buf)
+	r = NewControlFileReader(buf, false, false)
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
+	s.reader = bytes.NewBufferString(installerFile)
+	r := NewControlFileReader(s.reader, false, true)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, false, false, true)
+	c.Assert(err, IsNil)
+	err = w.Flush()
+	c.Assert(err, IsNil)
+
+	r = NewControlFileReader(buf, false, true)
+	stanza2, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Assert(stanza2, DeepEquals, stanza)
+}
+
 func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("Package"), Equals, "Package")
 	c.Check(canonicalCase("package"), Equals, "Package")
@@ -135,12 +160,23 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("packaGe-lIst"), Equals, "Package-List")
 }
 
+func (s *ControlFileSuite) TestLongFields(c *C) {
+	f, err := os.Open("long.stanza")
+	c.Assert(err, IsNil)
+	defer f.Close()
+
+	r := NewControlFileReader(f, false, false)
+	stanza, e := r.ReadStanza()
+	c.Assert(e, IsNil)
+	c.Assert(len(stanza["Provides"]), Equals, 586929)
+}
+
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
-		r := NewControlFileReader(reader)
+		r := NewControlFileReader(reader, false, false)
 		for {
-			s, e := r.ReadStanza(false)
+			s, e := r.ReadStanza()
 			if s == nil && e == nil {
 				break
 			}

deb/import.go

@@ -6,13 +6,13 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -32,6 +32,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
+				} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+					otherFiles = append(otherFiles, path)
 				}
 
 				return nil
@@ -46,6 +48,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
+			} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+				otherFiles = append(otherFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
 				failedFiles = append(failedFiles, location)
@@ -147,14 +151,34 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		// go over all the other files
 		for i := range files {
 			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+
+			_, err = os.Stat(sourceFile)
+			if err == nil {
+				files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+				if err == nil {
+					candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+				}
+			} else if os.IsNotExist(err) {
+				// if file is not present, try to find it in the pool
+				var (
+					err2  error
+					found bool
+				)
+
+				files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage)
+				if err2 != nil {
+					err = err2
+				} else if found {
+					// clear error, file is already in the package pool
+					err = nil
+				}
+			}
+
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
 				break
 			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
 		}
 		if err != nil {
 			// some files haven't been imported

deb/index_files.go

@@ -4,12 +4,13 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 type indexFiles struct {
@@ -20,18 +21,21 @@ type indexFiles struct {
 	tempDir          string
 	suffix           string
 	indexes          map[string]*indexFile
+	acquireByHash    bool
 }
 
 type indexFile struct {
-	parent       *indexFiles
+	parent        *indexFiles
-	discardable  bool
+	discardable   bool
-	compressable bool
+	compressable  bool
-	onlyGzip     bool
+	onlyGzip      bool
-	signable     bool
+	clearSign     bool
-	relativePath string
+	detachedSign  bool
-	tempFilename string
+	acquireByHash bool
-	tempFile     *os.File
+	relativePath  string
-	w            *bufio.Writer
+	tempFilename  string
+	tempFile      *os.File
+	w             *bufio.Writer
 }
 
 func (file *indexFile) BufWriter() (*bufio.Writer, error) {
@@ -74,14 +78,17 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	file.tempFile.Close()
 
 	exts := []string{""}
+	cksumExts := exts
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		cksumExts = exts
 		if file.onlyGzip {
 			exts = []string{".gz"}
+			cksumExts = []string{"", ".gz"}
 		}
 	}
 
-	for _, ext := range exts {
+	for _, ext := range cksumExts {
 		var checksumInfo utils.ChecksumInfo
 
 		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
@@ -91,11 +98,22 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
 	}
 
-	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+
+	err = file.parent.publishedStorage.MkDir(filedir)
 	if err != nil {
 		return fmt.Errorf("unable to create dir: %s", err)
 	}
 
+	if file.acquireByHash {
+		for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
+			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
+			if err != nil {
+				return fmt.Errorf("unable to create dir: %s", err)
+			}
+		}
+	}
+
 	for _, ext := range exts {
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
@@ -107,43 +125,111 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
 				filepath.Join(file.parent.basePath, file.relativePath+ext)
 		}
+
+		if file.acquireByHash {
+			sums := file.parent.generatedFiles[file.relativePath+ext]
+			for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
+				err = packageIndexByHash(file, ext, hash, sum)
+				if err != nil {
+					return fmt.Errorf("unable to build hash file: %s", err)
+				}
+			}
+		}
 	}
 
-	if file.signable && signer != nil {
+	if signer != nil {
-		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if file.detachedSign {
-		if err != nil {
+			err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
-			return fmt.Errorf("unable to detached sign file: %s", err)
+			if err != nil {
+				return fmt.Errorf("unable to detached sign file: %s", err)
+			}
+
+			if file.parent.suffix != "" {
+				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+					filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			}
+
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+				file.tempFilename+".gpg")
+			if err != nil {
+				return fmt.Errorf("unable to publish file: %s", err)
+			}
+
 		}
 
-		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if file.clearSign {
-		if err != nil {
+			err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-			return fmt.Errorf("unable to clearsign file: %s", err)
+			if err != nil {
-		}
+				return fmt.Errorf("unable to clearsign file: %s", err)
+			}
 
-		if file.parent.suffix != "" {
+			if file.parent.suffix != "" {
-			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
-				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+					filepath.Join(file.parent.basePath, "In"+file.relativePath)
-			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+			}
-				filepath.Join(file.parent.basePath, "In"+file.relativePath)
-		}
 
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
-			file.tempFilename+".gpg")
+				filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
+			if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
+				return fmt.Errorf("unable to publish file: %s", err)
-		}
+			}
-
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
-			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
 		}
 	}
 
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
+	src := filepath.Join(file.parent.basePath, file.relativePath)
+	indexfile := path.Base(src + ext)
+	src = src + file.parent.suffix + ext
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+	dst := filepath.Join(filedir, "by-hash", hash)
+	sumfilePath := filepath.Join(dst, sum)
+
+	// link already exists? do nothing
+	exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
+	}
+	if exists {
+		return nil
+	}
+
+	// create the link
+	err = file.parent.publishedStorage.HardLink(src, sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
+	}
+
+	// if a previous index file already exists exists, backup symlink
+	indexPath := filepath.Join(dst, indexfile)
+	oldIndexPath := filepath.Join(dst, indexfile+".old")
+	if exists, _ = file.parent.publishedStorage.FileExists(indexPath); exists {
+		// if exists, remove old symlink
+		if exists, _ = file.parent.publishedStorage.FileExists(oldIndexPath); exists {
+			var linkTarget string
+			linkTarget, err = file.parent.publishedStorage.ReadLink(oldIndexPath)
+			if err == nil {
+				// If we managed to resolve the link target: delete it. This is the
+				// oldest physical index file we no longer need. Once we drop our
+				// old symlink we'll essentially forget about it existing at all.
+				file.parent.publishedStorage.Remove(linkTarget)
+			}
+			file.parent.publishedStorage.Remove(oldIndexPath)
+		}
+		file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
+	}
+
+	// create symlink
+	err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
+	}
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -152,14 +238,15 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		tempDir:          tempDir,
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
+		acquireByHash:    acquireByHash,
 	}
 }
 
-func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false
 	}
-	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
+	key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -169,17 +256,21 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		} else {
 			if udeb {
 				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else if installer {
+				relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
 			} else {
 				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
 			}
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  false,
+			discardable:   false,
-			compressable: true,
+			compressable:  !installer,
-			signable:     false,
+			detachedSign:  installer,
-			relativePath: relativePath,
+			clearSign:     false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -208,11 +299,13 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  udeb,
+			discardable:   udeb,
-			compressable: false,
+			compressable:  false,
-			signable:     false,
+			detachedSign:  false,
-			relativePath: relativePath,
+			clearSign:     false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -237,12 +330,46 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  true,
+			discardable:   true,
-			compressable: true,
+			compressable:  true,
-			onlyGzip:     true,
+			onlyGzip:      true,
-			signable:     false,
+			detachedSign:  false,
-			relativePath: relativePath,
+			clearSign:     false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile {
+	if arch == ArchitectureSource {
+		udeb = false
+	}
+	key := fmt.Sprintf("lci-%s-%v", arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if udeb {
+			relativePath = fmt.Sprintf("Contents-udeb-%s", arch)
+		} else {
+			relativePath = fmt.Sprintf("Contents-%s", arch)
+		}
+
+		file = &indexFile{
+			parent:        files,
+			discardable:   true,
+			compressable:  true,
+			onlyGzip:      true,
+			detachedSign:  false,
+			clearSign:     false,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -256,19 +383,20 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 		parent:       files,
 		discardable:  false,
 		compressable: false,
-		signable:     true,
+		detachedSign: true,
+		clearSign:    true,
 		relativePath: "Release",
 	}
 }
 
-func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
 		progress.InitBar(int64(len(files.indexes)), false)
 		defer progress.ShutdownBar()
 	}
 
 	for _, file := range files.indexes {
-		err = file.Finalize(nil)
+		err = file.Finalize(signer)
 		if err != nil {
 			return
 		}

deb/list.go

@@ -5,8 +5,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // Dependency options
@@ -124,6 +124,14 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	return result, nil
 }
 
+// Has checks whether package is already in the list
+func (l *PackageList) Has(p *Package) bool {
+	key := l.keyFunc(p)
+	_, ok := l.packages[key]
+
+	return ok
+}
+
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
 	key := l.keyFunc(p)
@@ -441,18 +449,6 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		panic("list not indexed, can't search")
 	}
 
-	if dep.Relation == VersionDontCare {
-		for _, p := range l.providesIndex[dep.Pkg] {
-			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
-				searchResults = append(searchResults, p)
-
-				if !allMatches {
-					break
-				}
-			}
-		}
-	}
-
 	i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
 
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
@@ -468,6 +464,18 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		i++
 	}
 
+	if dep.Relation == VersionDontCare {
+		for _, p := range l.providesIndex[dep.Pkg] {
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
+			}
+		}
+	}
+
 	return
 }
 
@@ -511,15 +519,27 @@ func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencie
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				// dependency might have already been satisfied
+				if dependencyOptions&DepFollowAllVariants == 0 {
-				// with packages already been added
+					// dependency might have already been satisfied
-				if result.Search(dep, false) != nil {
+					// with packages already been added
-					continue
+					//
+					// when follow-all-variants is enabled, we need to try to expand anyway,
+					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					if result.Search(dep, false) != nil {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true))
+						}
+						continue
+					}
 				}
 
 				searchResults := l.Search(dep, true)
-				if searchResults != nil {
+				if len(searchResults) > 0 {
 					for _, p := range searchResults {
+						if result.Has(p) {
+							continue
+						}
+
 						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
 							progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
 						}

deb/local.go

@@ -2,12 +2,13 @@ package deb
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"log"
 	"sync"
 
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -93,46 +94,68 @@ func (repo *LocalRepo) RefKey() []byte {
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*LocalRepo
+	cache map[string]*LocalRepo
 }
 
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
-	result := &LocalRepoCollection{
+	return &LocalRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*LocalRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("L"))
+func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, unique bool) []*LocalRepo {
-	result.list = make([]*LocalRepo, 0, len(blobs))
+	result := []*LocalRepo(nil)
-
+	for _, r := range collection.cache {
-	for _, blob := range blobs {
+		if filter(r) {
-		r := &LocalRepo{}
+			result = append(result, r)
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding repo: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+		r := &LocalRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding local repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
-	for _, r := range collection.list {
+	_, err := collection.ByName(repo.Name)
-		if r.Name == repo.Name {
+
-			return fmt.Errorf("local repo with name %s already exists", repo.Name)
+	if err == nil {
-		}
+		return fmt.Errorf("local repo with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -167,58 +190,66 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 
 // ByName looks up repository by name
 func (collection *LocalRepoCollection) ByName(name string) (*LocalRepo, error) {
-	for _, r := range collection.list {
+	result := collection.search(func(r *LocalRepo) bool { return r.Name == name }, true)
-		if r.Name == name {
+	if len(result) == 0 {
-			return r, nil
+		return nil, fmt.Errorf("local repo with name %s not found", name)
-		}
 	}
-	return nil, fmt.Errorf("local repo with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
-	for _, r := range collection.list {
+	if r, ok := collection.cache[uuid]; ok {
-		if r.UUID == uuid {
+		return r, nil
-			return r, nil
-		}
 	}
-	return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+
+	key := (&LocalRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	r := &LocalRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &LocalRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding repo: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *LocalRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("L")))
 }
 
 // Drop removes remote repo from collection
 func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
-	repoPosition := -1
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
 		panic("local repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+	delete(collection.cache, repo.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/local_test.go

@@ -3,7 +3,7 @@ package deb
 import (
 	"errors"
 
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 
 	. "gopkg.in/check.v1"
 )
@@ -124,6 +124,11 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewLocalRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 

deb/package.go

@@ -1,14 +1,15 @@
 package deb
 
 import (
+	gocontext "context"
 	"encoding/json"
 	"fmt"
 	"path/filepath"
 	"strconv"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // Package is single instance of Debian package
@@ -26,6 +27,8 @@ type Package struct {
 	Provides []string
 	// Hash of files section
 	FilesHash uint64
+	// Is this package a dummy installer package
+	IsInstaller bool
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
@@ -43,9 +46,10 @@ type Package struct {
 
 // Package types
 const (
-	PackageTypeBinary = "deb"
+	PackageTypeBinary    = "deb"
-	PackageTypeUdeb   = "udeb"
+	PackageTypeUdeb      = "udeb"
-	PackageTypeSource = "source"
+	PackageTypeSource    = "source"
+	PackageTypeInstaller = "installer"
 )
 
 // Special arhictectures
@@ -168,6 +172,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package {
 	return p
 }
 
+// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file
+func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) {
+	p := &Package{
+		Name:         "installer",
+		Architecture: architecture,
+		IsInstaller:  true,
+		V06Plus:      true,
+		extra:        &Stanza{},
+		deps:         &PackageDependencies{},
+	}
+
+	files := make(PackageFiles, 0)
+	files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	for i := range files {
+		files[i].downloadPath = relPath
+
+		url := repo.PackageURL(files[i].DownloadURL()).String()
+		var size int64
+		size, err = d.GetLength(gocontext.TODO(), url)
+		if err != nil {
+			return nil, err
+		}
+
+		files[i].Checksums.Size = size
+	}
+
+	p.UpdateFiles(files)
+	return p, nil
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -509,6 +548,12 @@ func (p *Package) Stanza() (result Stanza) {
 		if len(sha512) > 0 {
 			result["Checksums-Sha512"] = strings.Join(sha512, "")
 		}
+	} else if p.IsInstaller {
+		sha256 := []string{}
+		for _, f := range p.Files() {
+			sha256 = append(sha256, fmt.Sprintf("%s  %s", f.Checksums.SHA256, f.Filename))
+		}
+		result[""] = strings.Join(sha256, "\n")
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -563,11 +608,7 @@ func (p *Package) Equals(p2 *Package) bool {
 
 // LinkFromPool links package file from pool to dist's pool location
 func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
-	prefix, component string, force bool) error {
+	prefix, relPath string, force bool) error {
-	poolDir, err := p.PoolDirectory()
-	if err != nil {
-		return err
-	}
 
 	for i, f := range p.Files() {
 		sourcePoolPath, err := f.GetPoolPath(packagePool)
@@ -575,7 +616,6 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
 		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
@@ -622,6 +662,7 @@ type PackageDownloadTask struct {
 	File         *PackageFile
 	Additional   []PackageDownloadTask
 	TempDownPath string
+	Done         bool
 }
 
 // DownloadList returns list of missing package files for download in format
@@ -629,14 +670,15 @@ type PackageDownloadTask struct {
 func (p *Package) DownloadList(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result []PackageDownloadTask, err error) {
 	result = make([]PackageDownloadTask, 0, 1)
 
-	for idx, f := range p.Files() {
+	files := p.Files()
-		verified, err := f.Verify(packagePool, checksumStorage)
+	for idx := range files {
+		verified, err := files[idx].Verify(packagePool, checksumStorage)
 		if err != nil {
 			return nil, err
 		}
 
 		if !verified {
-			result = append(result, PackageDownloadTask{File: &p.Files()[idx]})
+			result = append(result, PackageDownloadTask{File: &files[idx]})
 		}
 	}
 

deb/package_collection.go

@@ -5,8 +5,8 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 	"github.com/ugorji/go/codec"
 )
 
@@ -315,7 +315,7 @@ func (collection *PackageCollection) SearchSupported() bool {
 
 // SearchByKey finds package by exact key
 func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
 		pkg, err := collection.ByKey(key)

deb/package_collection_test.go

@@ -1,8 +1,8 @@
 package deb
 
 import (
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )

deb/package_files.go

@@ -9,8 +9,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 // PackageFile is a single file entry in package
@@ -27,7 +27,12 @@ type PackageFile struct {
 
 // Verify that package file is present and correct
 func (f *PackageFile) Verify(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (bool, error) {
-	return packagePool.Verify(f.PoolPath, f.Filename, &f.Checksums, checksumStorage)
+	generatedPoolPath, exists, err := packagePool.Verify(f.PoolPath, f.Filename, &f.Checksums, checksumStorage)
+	if exists && err == nil {
+		f.PoolPath = generatedPoolPath
+	}
+
+	return exists, err
 }
 
 // GetPoolPath returns path to the file in the pool
@@ -83,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
 
-func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+// ParseSumField populates PackageFiles by parsing given input
+func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) {
 	for _, line := range strings.Split(input, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
@@ -91,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 		}
 		parts := strings.Fields(line)
 
-		if len(parts) < 3 {
+		if withSize && len(parts) < 3 || !withSize && len(parts) < 2 {
 			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
 		}
 
-		size, err := strconv.ParseInt(parts[1], 10, 64)
+		var size int64
-		if err != nil {
+		var err error
-			return nil, fmt.Errorf("unable to parse size: %s", err)
+		if withSize {
+			size, err = strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse size: %s", err)
+			}
 		}
 
-		filename := filepath.Base(parts[len(parts)-1])
+		filename := parts[len(parts)-1]
+		if onlyBasePath {
+			filename = filepath.Base(filename)
+		}
 
 		found := false
 		pos := 0
@@ -128,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
 	var err error
 
-	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}

deb/package_files_test.go

@@ -4,9 +4,9 @@ import (
 	"io/ioutil"
 	"path/filepath"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/files"
+	"github.com/aptly-dev/aptly/files"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )

deb/package_test.go

@@ -6,7 +6,8 @@ import (
 	"path/filepath"
 	"regexp"
 
-	"github.com/smira/aptly/files"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/http"
 
 	. "gopkg.in/check.v1"
 )
@@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
+	s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza()
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
 }
 
+func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
+	downloader.ExpectResponse("", "udeb.list")
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza()
+	p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader)
+	c.Check(err, IsNil)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
+	c.Check(p.IsInstaller, Equals, true)
+	c.Check(p.Name, Equals, "installer")
+	c.Check(p.Version, Equals, "")
+	c.Check(p.Architecture, Equals, "i386")
+	c.Check(p.Files(), HasLen, 2)
+	c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs")
+	c.Check(p.Files()[0].Checksums.Size, Equals, int64(14))
+	c.Check(p.Files()[1].Filename, Equals, "./udeb.list")
+	c.Check(p.Files()[1].Checksums.Size, Equals, int64(9))
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
@@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -490,3 +514,6 @@ Size: 29188
 MD5sum: ae70341c4d96dcded89fa670bcfea31e
 SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
 SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`
+
+const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list`

deb/ppa.go

@@ -6,7 +6,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")

deb/ppa_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )

deb/publish.go

@@ -14,13 +14,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 )
 
 type repoSourceItem struct {
@@ -64,6 +64,9 @@ type PublishedRepo struct {
 
 	// True if repo is being re-published
 	rePublishing bool
+
+	// Provide index files per hash also
+	AcquireByHash bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -312,6 +315,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"Sources":              sources,
 		"Storage":              p.Storage,
 		"SkipContents":         p.SkipContents,
+		"AcquireByHash":        p.AcquireByHash,
 	})
 }
 
@@ -556,14 +560,16 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
+
+	legacyContentIndexes := map[string]*ContentsIndex{}
 
 	for component, list := range lists {
 		hadUdebs := false
 
 		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
-			indexes.PackageIndex(component, arch, false)
+			indexes.PackageIndex(component, arch, false, false)
 		}
 
 		if progress != nil {
@@ -579,45 +585,63 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				progress.AddBar(1)
 			}
 
-			matches := false
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					matches = true
+					hadUdebs = hadUdebs || pkg.IsUdeb
+
+					var relPath string
+					if !pkg.IsInstaller {
+						poolDir, err2 := pkg.PoolDirectory()
+						if err2 != nil {
+							return err2
+						}
+						relPath = filepath.Join("pool", component, poolDir)
+					} else {
+						relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
+					}
+
+					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite)
+					if err != nil {
+						return err
+					}
 					break
 				}
 			}
 
-			if matches {
+			// Start a db batch. If we fill contents data we'll need
-				hadUdebs = hadUdebs || pkg.IsUdeb
+			// to push each path of the package into the database.
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
+			// We'll want this batched so as to avoid an excessive
-				if err != nil {
+			// amount of write() calls.
-					return err
+			tempDB.StartBatch()
-				}
+			defer tempDB.FinishBatch()
-			}
 
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
-					if !p.SkipContents {
+					if !p.SkipContents && !pkg.IsInstaller {
 						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
+						qualifiedName := []byte(pkg.QualifiedName())
+						contents := pkg.Contents(packagePool, progress)
 
-						contentIndex := contentIndexes[key]
+						for _, contentIndexesMap := range []map[string]*ContentsIndex{contentIndexes, legacyContentIndexes} {
+							contentIndex := contentIndexesMap[key]
 
-						if contentIndex == nil {
+							if contentIndex == nil {
-							contentIndex = NewContentsIndex(tempDB)
+								contentIndex = NewContentsIndex(tempDB)
-							contentIndexes[key] = contentIndex
+								contentIndexesMap[key] = contentIndex
+							}
+
+							contentIndex.Push(qualifiedName, contents)
 						}
-
-						contentIndex.Push(pkg, packagePool, progress)
 					}
 
-					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller)
 					if err != nil {
 						return err
 					}
@@ -670,7 +694,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			// For all architectures, pregenerate .udeb indexes
 			for _, arch := range p.Architectures {
-				indexes.PackageIndex(component, arch, true)
+				indexes.PackageIndex(component, arch, true, false)
 			}
 		}
 
@@ -683,6 +707,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				if p.AcquireByHash {
+					release["Acquire-By-Hash"] = "yes"
+				}
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -690,7 +717,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
 				}
 
-				err = release.WriteTo(bufWriter, false, true)
+				err = release.WriteTo(bufWriter, false, true, false)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -698,11 +725,31 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		}
 	}
 
+	for _, arch := range p.Architectures {
+		for _, udeb := range []bool{true, false} {
+			index := legacyContentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
+			if index == nil || index.Empty() {
+				continue
+			}
+
+			var bufWriter *bufio.Writer
+			bufWriter, err = indexes.LegacyContentsIndex(arch, udeb).BufWriter()
+			if err != nil {
+				return fmt.Errorf("unable to generate contents index: %v", err)
+			}
+
+			_, err = index.WriteTo(bufWriter)
+			if err != nil {
+				return fmt.Errorf("unable to generate contents index: %v", err)
+			}
+		}
+	}
+
 	if progress != nil {
 		progress.Printf("Finalizing metadata files...\n")
 	}
 
-	err = indexes.FinalizeAll(progress)
+	err = indexes.FinalizeAll(progress, signer)
 	if err != nil {
 		return err
 	}
@@ -720,6 +767,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
+	if p.AcquireByHash {
+		release["Acquire-By-Hash"] = "yes"
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""
@@ -748,7 +798,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter, false, true)
+	err = release.WriteTo(bufWriter, false, true, false)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -809,28 +859,34 @@ type PublishedRepoCollection struct {
 
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
-	result := &PublishedRepoCollection{
+	return &PublishedRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("U"))
+func (collection *PublishedRepoCollection) loadList() {
-	result.list = make([]*PublishedRepo, 0, len(blobs))
+	if collection.list != nil {
+		return
+	}
+
+	blobs := collection.db.FetchByPrefix([]byte("U"))
+	collection.list = make([]*PublishedRepo, 0, len(blobs))
 
 	for _, blob := range blobs {
 		r := &PublishedRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding published repo: %s\n", err)
 		} else {
-			result.list = append(result.list, r)
+			collection.list = append(collection.list, r)
 		}
 	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
+	collection.loadList()
+
 	if collection.CheckDuplicate(repo) != nil {
 		return fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s already exists", repo.Storage, repo.Prefix, repo.Distribution)
 	}
@@ -846,6 +902,8 @@ func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 
 // CheckDuplicate verifies that there's no published repo with the same name
 func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution && r.Storage == repo.Storage {
 			return r
@@ -935,6 +993,8 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 // ByStoragePrefixDistribution looks up repository by storage, prefix & distribution
 func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage, prefix, distribution string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == prefix && r.Distribution == distribution && r.Storage == storage {
 			return r, nil
@@ -948,6 +1008,8 @@ func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage,
 
 // ByUUID looks up repository by uuid
 func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.UUID == uuid {
 			return r, nil
@@ -958,6 +1020,8 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 
 // BySnapshot looks up repository by snapshot source
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceSnapshot {
@@ -978,6 +1042,8 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 
 // ByLocalRepo looks up repository by local repo source
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceLocalRepo {
@@ -998,18 +1064,21 @@ func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*Publi
 
 // ForEach runs method for each repository
 func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("U"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &PublishedRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding published repo: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *PublishedRepoCollection) Len() int {
+	collection.loadList()
+
 	return len(collection.list)
 }
 
@@ -1017,6 +1086,8 @@ func (collection *PublishedRepoCollection) Len() int {
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
 	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
 
+	collection.loadList()
+
 	var err error
 	referencedFiles := map[string][]string{}
 
@@ -1097,7 +1168,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
-	force bool) error {
+	force, skipCleanup bool) error {
+
+	collection.loadList()
+
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1128,13 +1202,16 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 
 	err = repo.RemoveFiles(publishedStorageProvider, removePrefix, removePoolComponents, progress)
 	if err != nil {
-		return err
+		if !force {
+			return fmt.Errorf("published files removal failed, use -force-drop to override: %s", err)
+		}
+		// ignore error with -force-drop
 	}
 
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
-	if len(cleanComponents) > 0 {
+	if !skipCleanup && len(cleanComponents) > 0 {
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {

deb/publish_test.go

@@ -8,9 +8,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/files"
+	"github.com/aptly-dev/aptly/files"
 	"github.com/ugorji/go/codec"
 
 	. "gopkg.in/check.v1"
@@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 
-	cfr := NewControlFileReader(rf)
+	cfr := NewControlFileReader(rf, true, false)
-	st, err := cfr.ReadStanza(true)
+	st, err := cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(pf)
+	cfr = NewControlFileReader(pf, false, false)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza(false)
+		st, err = cfr.ReadStanza()
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza(false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
 	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(drf)
+	cfr = NewControlFileReader(drf, true, false)
-	st, err = cfr.ReadStanza(true)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")
@@ -756,7 +756,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -776,10 +776,48 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, false)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2SkipCleanup(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, true)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -794,7 +832,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -816,7 +854,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/reflist_test.go

@@ -3,7 +3,7 @@ package deb
 import (
 	"errors"
 
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
 
 	. "gopkg.in/check.v1"
 )

deb/remote.go

@@ -2,6 +2,8 @@ package deb
 
 import (
 	"bytes"
+	gocontext "context"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -14,12 +16,12 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/http"
+	"github.com/aptly-dev/aptly/http"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -67,6 +69,8 @@ type RemoteRepo struct {
 	DownloadSources bool
 	// Should we download .udebs?
 	DownloadUdebs bool
+	// Should we download installer files?
+	DownloadInstaller bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -77,16 +81,17 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:            uuid.New(),
+		UUID:              uuid.New(),
-		Name:            name,
+		Name:              name,
-		ArchiveRoot:     archiveRoot,
+		ArchiveRoot:       archiveRoot,
-		Distribution:    distribution,
+		Distribution:      distribution,
-		Components:      components,
+		Components:        components,
-		Architectures:   architectures,
+		Architectures:     architectures,
-		DownloadSources: downloadSources,
+		DownloadSources:   downloadSources,
-		DownloadUdebs:   downloadUdebs,
+		DownloadUdebs:     downloadUdebs,
+		DownloadInstaller: downloadInstaller,
 	}
 
 	err := result.prepare()
@@ -112,6 +117,12 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 	return result, nil
 }
 
+// SetArchiveRoot of remote repo
+func (repo *RemoteRepo) SetArchiveRoot(archiveRoot string) {
+	repo.ArchiveRoot = archiveRoot
+	repo.prepare()
+}
+
 func (repo *RemoteRepo) prepare() error {
 	var err error
 
@@ -133,6 +144,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadUdebs {
 		srcFlag += " [udeb]"
 	}
+	if repo.DownloadInstaller {
+		srcFlag += " [installer]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -236,6 +250,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
 	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
 }
 
+// InstallerPath returns path of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -252,13 +272,13 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 
 	if verifier == nil {
 		// 0. Just download release file to temporary URL
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 	} else {
 		// 1. try InRelease file
-		inrelease, err = http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		inrelease, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("InRelease").String())
 		if err != nil {
 			goto splitsignature
 		}
@@ -280,12 +300,12 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 
 	splitsignature:
 		// 2. try Release + Release.gpg
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 
-		releasesig, err = http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		releasesig, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release.gpg").String())
 		if err != nil {
 			return err
 		}
@@ -304,8 +324,8 @@ ok:
 
 	defer release.Close()
 
-	sreader := NewControlFileReader(release)
+	sreader := NewControlFileReader(release, true, false)
-	stanza, err := sreader.ReadStanza(true)
+	stanza, err := sreader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -391,7 +411,10 @@ ok:
 		return err
 	}
 
-	delete(stanza, "SHA512")
+	err = parseSums("SHA512", func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return err
+	}
 
 	repo.Meta = stanza
 
@@ -399,7 +422,7 @@ ok:
 }
 
 // DownloadPackageIndexes downloads & parses package index files
-func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
 	ignoreMismatch bool, maxTries int) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
@@ -410,39 +433,85 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""})
 		if repo.DownloadSources {
-			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture})
 				if repo.DownloadUdebs {
-					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture})
+				}
+				if repo.DownloadInstaller {
+					packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture})
 				}
 			}
 			if repo.DownloadSources {
-				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"})
 			}
 		}
 	}
 
 	for _, info := range packagesPaths {
-		path, kind := info[0], info[1]
+		path, kind, component, architecture := info[0], info[1], info[2], info[3]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+
+		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
-			return err
+			if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
+				// checking if gpg file is only needed when checksums matches are required.
+				// otherwise there actually has been no candidate found and we can continue
+				if ignoreMismatch {
+					continue
+				}
+
+				// some repos do not have installer hashsum file listed in release file but provide a separate gpg file
+				hashsumPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path}).String()
+				packagesFile, err = http.DownloadTemp(gocontext.TODO(), d, hashsumPath)
+				if err != nil {
+					if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+						// installer files are not available in all components and architectures
+						// so ignore it if not found
+						continue
+					}
+
+					return err
+				}
+
+				if verifier != nil {
+					hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
+					var filesig *os.File
+					filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
+					if err != nil {
+						return err
+					}
+
+					err = verifier.VerifyDetachedSignature(filesig, packagesFile, false)
+					if err != nil {
+						return err
+					}
+
+					_, err = packagesFile.Seek(0, 0)
+				}
+
+				packagesReader = packagesFile
+			}
+
+			if err != nil {
+				return err
+			}
 		}
 		defer packagesFile.Close()
 
 		stat, _ := packagesFile.Stat()
 		progress.InitBar(stat.Size(), true)
 
-		sreader := NewControlFileReader(packagesReader)
+		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
 		for {
-			stanza, err := sreader.ReadStanza(false)
+			stanza, err := sreader.ReadStanza()
 			if err != nil {
 				return err
 			}
@@ -464,15 +533,18 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				if err != nil {
 					return err
 				}
-			}
+			} else if kind == PackageTypeInstaller {
-			err = repo.packageList.Add(p)
+				p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d)
-			if err != nil {
+				if err != nil {
-				if _, ok := err.(*PackageConflictError); ok {
-					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
-				} else {
 					return err
 				}
 			}
+			err = repo.packageList.Add(p)
+			if _, ok := err.(*PackageConflictError); ok {
+				progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+			} else if err != nil {
+				return err
+			}
 		}
 
 		progress.ShutdownBar()
@@ -592,7 +664,7 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
 			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
-			var repo11 struct {
+			var repo11 struct { // nolint: maligned
 				UUID             string
 				Name             string
 				ArchiveRoot      string
@@ -644,46 +716,68 @@ func (repo *RemoteRepo) RefKey() []byte {
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*RemoteRepo
+	cache map[string]*RemoteRepo
 }
 
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
-	result := &RemoteRepoCollection{
+	return &RemoteRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*RemoteRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("R"))
+func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, unique bool) []*RemoteRepo {
-	result.list = make([]*RemoteRepo, 0, len(blobs))
+	result := []*RemoteRepo(nil)
-
+	for _, r := range collection.cache {
-	for _, blob := range blobs {
+		if filter(r) {
-		r := &RemoteRepo{}
+			result = append(result, r)
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+		r := &RemoteRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding remote repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
-	for _, r := range collection.list {
+	_, err := collection.ByName(repo.Name)
-		if r.Name == repo.Name {
+
-			return fmt.Errorf("mirror with name %s already exists", repo.Name)
+	if err == nil {
-		}
+		return fmt.Errorf("mirror with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -718,58 +812,65 @@ func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo) error {
 
 // ByName looks up repository by name
 func (collection *RemoteRepoCollection) ByName(name string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
+	result := collection.search(func(r *RemoteRepo) bool { return r.Name == name }, true)
-		if r.Name == name {
+	if len(result) == 0 {
-			return r, nil
+		return nil, fmt.Errorf("mirror with name %s not found", name)
-		}
 	}
-	return nil, fmt.Errorf("mirror with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *RemoteRepoCollection) ByUUID(uuid string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
+	if r, ok := collection.cache[uuid]; ok {
-		if r.UUID == uuid {
+		return r, nil
-			return r, nil
-		}
 	}
-	return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+
+	key := (&RemoteRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	r := &RemoteRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
-	for _, r := range collection.list {
+		r := &RemoteRepo{}
-		err = handler(r)
+		if err := r.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding mirror: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *RemoteRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("R")))
 }
 
 // Drop removes remote repo from collection
 func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
-	repoPosition := -1
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
 		panic("repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+	delete(collection.cache, repo.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/remote_test.go

@@ -7,13 +7,13 @@ import (
 	"os"
 	"sort"
 
-	"github.com/smira/aptly/aptly"
+	"github.com/aptly-dev/aptly/aptly"
-	"github.com/smira/aptly/console"
+	"github.com/aptly-dev/aptly/console"
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/files"
+	"github.com/aptly-dev/aptly/files"
-	"github.com/smira/aptly/http"
+	"github.com/aptly-dev/aptly/http"
-	"github.com/smira/aptly/pgp"
+	"github.com/aptly-dev/aptly/pgp"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
@@ -88,8 +88,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.NewOpenDB(c.MkDir())
@@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 
 	s.repo.DownloadSources = true
 	s.repo.DownloadUdebs = true
+	s.repo.DownloadInstaller = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
 	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
+func (s *RemoteRepoSuite) TestInstallerPath(c *C) {
+	c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS")
+}
+
 func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
 	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
 }
@@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -333,6 +338,49 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(s.repo.packageRefs, NotNil)
 }
 
+func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
+	s.repo.Architectures = []string{"i386"}
+	s.repo.DownloadInstaller = true
+
+	err := s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/SHA256SUMS", exampleInstallerHashSumFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST", exampleInstallerManifestFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile)))
+	c.Check(queue, HasLen, 2)
+
+	q := make([]string, 2)
+	for i := range q {
+		q[i] = queue[i].File.DownloadURL()
+	}
+	sort.Strings(q)
+	c.Check(q[0], Equals, "dists/squeeze/main/installer-i386/current/images/MANIFEST")
+	c.Check(q[1], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
+	c.Assert(err, IsNil)
+	c.Check(pkg.Name, Equals, "installer")
+}
+
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.repo.Architectures = []string{"i386"}
 	s.repo.DownloadSources = true
@@ -347,7 +395,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -391,7 +439,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -416,7 +464,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -440,7 +488,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -468,7 +516,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -490,7 +538,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -521,7 +569,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -567,7 +615,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -593,7 +641,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -628,7 +676,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -646,16 +694,21 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewRemoteRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -676,7 +729,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -698,10 +751,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
@@ -874,4 +927,31 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
 `
 
+const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a  ./MANIFEST
+`
+
+const exampleInstallerManifestFile = `cdrom/debian-cd_info.tar.gz     -- isolinux config files for CD
+cdrom/gtk/debian-cd_info.tar.gz -- isolinux help screens for CD (graphical)
+cdrom/gtk/initrd.gz             -- initrd for use with isolinux to build a CD (graphical)
+cdrom/gtk/vmlinuz               -- kernel for use with isolinux to build a CD (graphical)
+cdrom/initrd.gz                 -- initrd for use with isolinux to build a CD
+cdrom/vmlinuz                   -- kernel for use with isolinux to build a CD
+cdrom/xen/debian.cfg            -- example Xen configuration
+cdrom/xen/initrd.gz             -- initrd for installing under Xen
+cdrom/xen/vmlinuz               -- kernel image for installing under Xen
+hd-media/boot.img.gz            -- 1 gb image (compressed) for USB memory stick
+hd-media/gtk/initrd.gz          -- for use on USB memory sticks
+hd-media/gtk/vmlinuz            -- for use on USB memory sticks
+hd-media/initrd.gz              -- for use on USB memory sticks
+hd-media/vmlinuz                -- for use on USB memory sticks
+netboot/debian-installer        -- PXE boot directory for tftp server
+netboot/gtk/debian-installer    -- PXE boot directory for tftp server (graphical installer)
+netboot/gtk/mini.iso            -- not so tiny CD image that boots the graphical netboot installer
+netboot/gtk/netboot.tar.gz      -- tarball of PXE boot directory (graphical installer)
+netboot/mini.iso                -- tiny CD image that boots the netboot installer
+netboot/netboot.tar.gz          -- tarball of PXE boot directory
+netboot/xen/debian.cfg          -- example Xen configuration
+netboot/xen/initrd.gz           -- initrd for installing under Xen
+netboot/xen/vmlinuz             -- kernel image for installing under Xen`
+
 const exampleSourcesFile = sourcePackageMeta

deb/snapshot.go

@@ -10,9 +10,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/smira/aptly/database"
+	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/aptly/utils"
+	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -173,50 +173,36 @@ func (s *Snapshot) Decode(input []byte) error {
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
 type SnapshotCollection struct {
 	*sync.RWMutex
-	db   database.Storage
+	db    database.Storage
-	list []*Snapshot
+	cache map[string]*Snapshot
 }
 
 // NewSnapshotCollection loads Snapshots from DB and makes up collection
 func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
-	result := &SnapshotCollection{
+	return &SnapshotCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   map[string]*Snapshot{},
 	}
-
-	blobs := db.FetchByPrefix([]byte("S"))
-	result.list = make([]*Snapshot, 0, len(blobs))
-
-	for _, blob := range blobs {
-		s := &Snapshot{}
-		if err := s.Decode(blob); err != nil {
-			log.Printf("Error decoding snapshot: %s\n", err)
-		} else {
-			result.list = append(result.list, s)
-		}
-	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *SnapshotCollection) Add(snapshot *Snapshot) error {
-	for _, s := range collection.list {
+	_, err := collection.ByName(snapshot.Name)
-		if s.Name == snapshot.Name {
+	if err == nil {
-			return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
+		return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
-		}
 	}
 
-	err := collection.Update(snapshot)
+	err = collection.Update(snapshot)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, snapshot)
+	collection.cache[snapshot.UUID] = snapshot
 	return nil
 }
 
-// Update stores updated information about repo in DB
+// Update stores updated information about snapshot in DB
 func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	err := collection.db.Put(snapshot.Key(), snapshot.Encode())
 	if err != nil {
@@ -239,83 +225,132 @@ func (collection *SnapshotCollection) LoadComplete(snapshot *Snapshot) error {
 	return snapshot.packageRefs.Decode(encoded)
 }
 
-// ByName looks up snapshot by name
+func (collection *SnapshotCollection) search(filter func(*Snapshot) bool, unique bool) []*Snapshot {
-func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
+	result := []*Snapshot(nil)
-	for _, s := range collection.list {
+	for _, s := range collection.cache {
-		if s.Name == name {
+		if filter(s) {
-			return s, nil
+			result = append(result, s)
 		}
 	}
+
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+			return nil
+		}
+
+		if filter(s) {
+			if _, exists := collection.cache[s.UUID]; !exists {
+				collection.cache[s.UUID] = s
+				result = append(result, s)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return result
+}
+
+// ByName looks up snapshot by name
+func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
+	result := collection.search(func(s *Snapshot) bool { return s.Name == name }, true)
+	if len(result) > 0 {
+		return result[0], nil
+	}
+
 	return nil, fmt.Errorf("snapshot with name %s not found", name)
 }
 
 // ByUUID looks up snapshot by UUID
 func (collection *SnapshotCollection) ByUUID(uuid string) (*Snapshot, error) {
-	for _, s := range collection.list {
+	if s, ok := collection.cache[uuid]; ok {
-		if s.UUID == uuid {
+		return s, nil
-			return s, nil
-		}
 	}
-	return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+
+	key := (&Snapshot{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Snapshot{}
+	err = s.Decode(value)
+
+	if err == nil {
+		collection.cache[s.UUID] = s
+	}
+
+	return s, err
 }
 
 // ByRemoteRepoSource looks up snapshots that have specified RemoteRepo as a source
 func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // ByLocalRepoSource looks up snapshots that have specified LocalRepo as a source
 func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // BySnapshotSource looks up snapshots that have specified snapshot as a source
 func (collection *SnapshotCollection) BySnapshotSource(snapshot *Snapshot) []*Snapshot {
-	var result []*Snapshot
+	return collection.search(func(s *Snapshot) bool {
-
+		return s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID)
-	for _, s := range collection.list {
+	}, false)
-		if s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
 }
 
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) error {
-	var err error
+	return collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
-	for _, s := range collection.list {
+		s := &Snapshot{}
-		err = handler(s)
+		if err := s.Decode(blob); err != nil {
-		if err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
-			return err
+			return nil
 		}
-	}
+
-	return err
+		return handler(s)
+	})
 }
 
 // ForEachSorted runs method for each snapshot following some sort order
 func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error {
-	sorter, err := newSnapshotSorter(sortMethod, collection)
+	blobs := collection.db.FetchByPrefix([]byte("S"))
+	list := make([]*Snapshot, 0, len(blobs))
+
+	for _, blob := range blobs {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+		} else {
+			list = append(list, s)
+		}
+	}
+
+	sorter, err := newSnapshotSorter(sortMethod, list)
 	if err != nil {
 		return err
 	}
 
-	for _, i := range sorter.list {
+	for _, s := range sorter.list {
-		err = handler(collection.list[i])
+		err = handler(s)
 		if err != nil {
 			return err
 		}
@@ -327,26 +362,16 @@ func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler f
 // Len returns number of snapshots in collection
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("S")))
 }
 
 // Drop removes snapshot from collection
 func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
-	snapshotPosition := -1
+	if _, err := collection.db.Get(snapshot.Key()); err == database.ErrNotFound {
-
-	for i, s := range collection.list {
-		if s == snapshot {
-			snapshotPosition = i
-			break
-		}
-	}
-
-	if snapshotPosition == -1 {
 		panic("snapshot not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[snapshotPosition], collection.list =
+	delete(collection.cache, snapshot.UUID)
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
 	err := collection.db.Delete(snapshot.Key())
 	if err != nil {
@@ -363,13 +388,12 @@ const (
 )
 
 type snapshotSorter struct {
-	list       []int
+	list       []*Snapshot
-	collection *SnapshotCollection
 	sortMethod int
 }
 
-func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snapshotSorter, error) {
+func newSnapshotSorter(sortMethod string, list []*Snapshot) (*snapshotSorter, error) {
-	s := &snapshotSorter{collection: collection}
+	s := &snapshotSorter{list: list}
 
 	switch sortMethod {
 	case "time", "Time":
@@ -380,11 +404,6 @@ func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snap
 		return nil, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
 	}
 
-	s.list = make([]int, len(collection.list))
-	for i := range s.list {
-		s.list[i] = i
-	}
-
 	sort.Sort(s)
 
 	return s, nil
@@ -397,9 +416,9 @@ func (s *snapshotSorter) Swap(i, j int) {
 func (s *snapshotSorter) Less(i, j int) bool {
 	switch s.sortMethod {
 	case SortName:
-		return s.collection.list[s.list[i]].Name < s.collection.list[s.list[j]].Name
+		return s.list[i].Name < s.list[j].Name
 	case SortTime:
-		return s.collection.list[s.list[i]].CreatedAt.Before(s.collection.list[s.list[j]].CreatedAt)
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
 	}
 	panic("unknown sort method")
 }