Merge pull request #683 from smira/545-download-contxt

Use Go context to abort gracefully mirror updates

.gitignore

@@ -34,3 +34,5 @@ man/aptly.1.html
 man/aptly.1.ronn
 
 .goxc.local.json
+
+system/env/

.goxc.json

@@ -10,17 +10,17 @@
 		"bintray"
     ],
     "TaskSettings": {
-        "deb": {
+        "debs": {
             "metadata": {
                 "maintainer": "Andrey Smirnov",
-                "maintainerEmail": "me@smira.ru",
+                "maintainer-email": "me@smira.ru",
                 "description": "Debian repository management tool"
             },
             "metadata-deb": {
                 "License": "MIT",
                 "Homepage": "https://www.aptly.info/",
-                "Recommends": "bzip2, graphviz, xz-utils",
+                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-                "Depends": ""
+				"Suggests": "graphviz"
             },
             "other-mapped-files": {
                 "/": "root/"

.travis.yml

@@ -1,12 +1,15 @@
-sudo: false
+dist: trusty
+sudo: required
 
 language: go
 
 go:
-  - 1.6
+  - 1.7.x
-  - 1.7
+  - 1.8.x
-  - 1.8
+  - 1.9.x
-  - tip
+  - master
+
+go_import_path: github.com/smira/aptly
 
 addons:
   apt:
@@ -20,14 +23,11 @@ env:
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - virtualenv env
+  - virtualenv system/env
-  - . env/bin/activate
+  - . system/env/bin/activate
   - pip install six packaging appdirs
   - pip install -U pip setuptools
-  - pip install boto requests requests-unixsocket python-swiftclient
+  - pip install -r system/requirements.txt
-  - mkdir -p $GOPATH/src/github.com/smira
-  - ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/smira || true
-  - cd $GOPATH/src/github.com/smira/aptly
   - make version
 install:
   - make prepare
@@ -36,7 +36,7 @@ script: make travis
 
 matrix:
     allow_failures:
-        - go: tip
+        - go: master
 
 notifications:
   webhooks:

AUTHORS

@@ -26,3 +26,7 @@ List of contributors, in chronological order:
 * Harald Sitter (https://github.com/apachelogger)
 * Johannes Layher (https://github.com/jola5)
 * Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)
+* Matt Martyn (https://github.com/MMartyn)
+* Ludovico Cavedon (https://github.com/cavedon)

CONTRIBUTING.md

@@ -0,0 +1,239 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+
+### List of Repositories
+
+* [smira/aptly](https://github.com/smira/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/smira/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/smira/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/smira/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly/info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/smira/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Your Fist Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+## Development Setup
+
+This section describes local setup to start contributing to aptly source.
+
+### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.  
+
+### Forking and Cloning
+
+As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
+
+    mkdir -p ~/go/src/github.com/smira
+    cd ~/go/src/github.com/smira
+    git clone git@github.com:smira/aptly.git
+    cd aptly
+
+For main repo under your GitHub user and add it as another Git remote:
+
+    git remote add <user> git@github.com:<user>/aptly.git
+
+That way you can continue to build project as is (you don't need to adjust import paths), but you would need
+to specify your remote name when pushing branches:
+
+    git push <user> <your-branch>
+
+### Dependencies
+
+You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+
+    make prepare dev system/env
+
+This is usually one-time action.
+
+### Building
+
+If you want to build aptly binary from your current source tree, run:
+
+    make install
+
+This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
+
+    aptly
+
+Or, if it's not on your path:
+
+    ~/go/bin/aptly
+
+### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### Style Checks
+
+Style checks could be run with:
+
+    make check
+
+aptly is using [gometalinter](https://github.com/alecthomas/gometalinter) to run style checks on Go code. Configuration
+for the linter could be found in [linter.json](linter.json) file. Running linters might take considerable amount of time
+unfortunately, but usually warning reported by linters hint at real code issues.
+
+Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
+
+### Vendored Code
+
+aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
+repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
+packages over packages in `$GOPATH`.
+
+If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
+Usually all you need is `dep ensure` or `dep ensure -update`.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash Completion
+
+Bash completion for aptly resides in the same repo under in [bash_completion.d/aptly](bash_completion.d/aptly). It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.
+
+## Design
+
+This section requires future work.
+
+*TBD*
+
+### Database
+
+### Package Pool
+
+### Package
+
+### PackageList, PackageRefList
+
+### LocalRepo, RemoteRepo, Snapshot
+
+### PublishedRepository
+
+### Context
+
+### Collections, CollectionFactory

Gopkg.lock

@@ -0,0 +1,216 @@
+memo = "57879f27cc9f82276b92ed638fbc04122c3793ed4a16bea668c9fbfda280c280"
+
+[[projects]]
+  name = "github.com/AlekSi/pointer"
+  packages = ["."]
+  revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/DisposaBoy/JsonConfigReader"
+  packages = ["."]
+  revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
+
+[[projects]]
+  name = "github.com/awalterschulze/gographviz"
+  packages = [".","ast","parser","scanner","token"]
+  revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
+  version = "v1.0"
+
+[[projects]]
+  name = "github.com/aws/aws-sdk-go"
+  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
+  revision = "c652f9369083515c3ddf1fbaf6df68da2c101545"
+  version = "v1.12.1"
+
+[[projects]]
+  name = "github.com/cheggaaa/pb"
+  packages = ["."]
+  revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
+  version = "v1.0.10"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/gin-contrib/sse"
+  packages = ["."]
+  revision = "22d885f9ecc78bf4ee5d72b937e4bbcdc58e8cae"
+
+[[projects]]
+  name = "github.com/gin-gonic/gin"
+  packages = [".","binding","render"]
+  revision = "d459835d2b077e44f7c9b453505ee29881d5d12d"
+  version = "v1.2"
+
+[[projects]]
+  name = "github.com/go-ini/ini"
+  packages = ["."]
+  revision = "1730955e3146956d6a087861380f9b4667ed5071"
+  version = "v1.26.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/protobuf"
+  packages = ["proto"]
+  revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  revision = "553a641470496b2327abcac10b36396bd98e45c9"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/h2non/filetype"
+  packages = ["matchers"]
+  revision = "0df83c38d14ff5f653d419d480eaac286ccbc823"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/jlaffaye/ftp"
+  packages = ["."]
+  revision = "7b85eb4638a2c0473acefcfb929a98f879c15c86"
+
+[[projects]]
+  name = "github.com/jmespath/go-jmespath"
+  packages = ["."]
+  revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9"
+  version = "0.2.2"
+
+[[projects]]
+  name = "github.com/mattn/go-isatty"
+  packages = ["."]
+  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
+  version = "v0.0.3"
+
+[[projects]]
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
+  version = "v0.0.2"
+
+[[projects]]
+  name = "github.com/mattn/go-shellwords"
+  packages = ["."]
+  revision = "005a0944d84452842197c2108bd9168ced206f78"
+  version = "v1.0.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mkrautz/goar"
+  packages = ["."]
+  revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mxk/go-flowrate"
+  packages = ["flowrate"]
+  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/ncw/swift"
+  packages = [".","swifttest"]
+  revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/commander"
+  packages = ["."]
+  revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/flag"
+  packages = ["."]
+  revision = "695ea5e84e76dea7c8656e43c384e54b32aa1b2a"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-aws-auth"
+  packages = ["."]
+  revision = "0070896e9d7f4f9f2d558532b2d896ce2239992a"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-ftp-protocol"
+  packages = ["protocol"]
+  revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-uuid"
+  packages = ["uuid"]
+  revision = "ed3ca8a15a931b141440a7e98e4f716eec255f7d"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-xz"
+  packages = ["."]
+  revision = "0c531f070014e218b21f3cfca801cc992d52726d"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/lzma"
+  packages = ["."]
+  revision = "7f0af6269940baa2c938fabe73e0d7ba41205683"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/syndtr/goleveldb"
+  packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
+  revision = "549b6d6b1c0419617182954dd77770f2e2685ed5"
+
+[[projects]]
+  name = "github.com/ugorji/go"
+  packages = ["codec"]
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/wsxiaoys/terminal"
+  packages = ["color"]
+  revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["cast5","openpgp","openpgp/armor","openpgp/clearsign","openpgp/elgamal","openpgp/errors","openpgp/packet","openpgp/s2k","ssh/terminal"]
+  revision = "459e26527287adbc2adcc5d0d49abff9a5f315a7"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "99f16d856c9836c42d24e7ab64ea72916925fa97"
+
+[[projects]]
+  branch = "v1"
+  name = "gopkg.in/check.v1"
+  packages = ["."]
+  revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
+
+[[projects]]
+  name = "gopkg.in/go-playground/validator.v8"
+  packages = ["."]
+  revision = "5f1438d3fca68893a817e4a66806cea46a9e4ebf"
+  version = "v8.18.2"
+
+[[projects]]
+  name = "gopkg.in/h2non/filetype.v1"
+  packages = ["types"]
+  revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
+  version = "v1.0.1"
+
+[[projects]]
+  branch = "v2"
+  name = "gopkg.in/yaml.v2"
+  packages = ["."]
+  revision = "eb3733d160e74a9c7e442f435eb3bea458e1d19f"

Gopkg.toml

@@ -0,0 +1,28 @@
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/mkrautz/goar"
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/smira/go-uuid"
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/smira/go-xz"
+
+[[dependencies]]
+  name = "github.com/ugorji/go"
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[dependencies]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+
+[[dependencies]]
+  branch = "master"
+  name = "golang.org/x/sys"
+
+[[dependencies]]
+  branch = "v1"
+  name = "gopkg.in/check.v1"

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
 VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files http query swift s3 utils
+PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python
 TESTS?=
 BINPATH?=$(GOPATH)/bin
@@ -35,16 +35,26 @@ coverage: coverage.out
 	go tool cover -html=coverage.out
 	rm -f coverage.out
 
-check:
+check: system/env
-	gometalinter --vendor --vendored-linters --config=linter.json ./...
+	if [ -x travis_wait ]; then \
+		travis_wait gometalinter --config=linter.json ./...; \
+	else \
+		gometalinter --config=linter.json ./...; \
+	fi
+	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
 
 install:
 	go install -v -ldflags "-X main.Version=$(VERSION)"
 
-system-test: install
+system/env: system/requirements.txt
+	rm -rf system/env
+	virtualenv system/env
+	system/env/bin/pip install -r system/requirements.txt
+
+system-test: install system/env
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
+	. system/env/bin/activate && APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
 
 travis: $(TRAVIS_TARGET) check system-test
 
@@ -58,20 +68,11 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-src-package:
-	rm -rf aptly-$(VERSION)
-	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
-	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
-	mkdir -p aptly-$(VERSION)/bash_completion.d
-	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
-	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
-	rm -rf aptly-$(VERSION)
-
 goxc:
 	rm -rf root/
 	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
 	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	cp bash_completion.d/aptly root/etc/bash_completion.d
 	gzip root/usr/share/man/man1/aptly.1
 	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
 

README.rst

@@ -2,11 +2,11 @@
 aptly
 =====
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
+.. image:: https://api.travis-ci.org/smira/aptly.svg?branch=master
     :target: https://travis-ci.org/smira/aptly
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
+.. image:: https://coveralls.io/repos/smira/aptly/badge.svg?branch=master
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+    :target: https://coveralls.io/r/smira/aptly?branch=master
 
 .. image:: https://badges.gitter.im/Join Chat.svg
     :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
@@ -19,7 +19,7 @@ Aptly is a swiss army knife for Debian repository management.
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
 Aptly features: ("+" means planned features)
@@ -42,7 +42,7 @@ Current limitations:
 Download
 --------
 
-To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
     deb http://repo.aptly.info/ squeeze main
 
@@ -64,7 +64,7 @@ If you would like to use nightly builds (unstable), please use following reposit
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running (go 1.6+ required)::
+If you have Go environment set up, you can build aptly from source by running (go 1.7+ required)::
 
     mkdir -p $GOPATH/src/github.com/smira/aptly
     git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
@@ -73,6 +73,11 @@ If you have Go environment set up, you can build aptly from source by running (g
 
 Binary would be installed to ```$GOPATH/bin/aptly``.
 
+Contributing
+------------
+
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+
 Integrations
 ------------
 

api/api.go

@@ -23,11 +23,18 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+type dbRequestKind int
+
 const (
-	acquiredb = iota
+	acquiredb dbRequestKind = iota
 	releasedb
 )
 
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
 // Flushes all collections which cache in-memory objects
 func flushColections() {
 	// lock everything to eliminate in-progress calls
@@ -52,50 +59,48 @@ func flushColections() {
 }
 
 // Periodically flushes CollectionFactory to free up memory used by
-// collections, flushing caches. If the two channels are provided,
+// collections, flushing caches.
-// they are used to acquire and release the database.
 //
 // Should be run in goroutine!
-func cacheFlusher(requests chan int, acks chan error) {
+func cacheFlusher() {
 	ticker := time.Tick(15 * time.Minute)
 
 	for {
 		<-ticker
 
-		// if aptly API runs in -no-lock mode,
+		flushColections()
-		// caches are flushed when DB is closed anyway, no need
-		// to flush them here
-		if requests == nil {
-			flushColections()
-		}
 	}
 }
 
-// Acquire database lock and release it when not needed anymore. Two
+// Acquire database lock and release it when not needed anymore.
-// channels must be provided. The first one is to receive requests to
-// acquire/release the database and the second one is to send acks.
 //
 // Should be run in a goroutine!
-func acquireDatabase(requests chan int, acks chan error) {
+func acquireDatabase(requests <-chan dbRequest) {
 	clients := 0
-	for {
+	for request := range requests {
-		request := <-requests
+		var err error
-		switch request {
+
+		switch request.kind {
 		case acquiredb:
 			if clients == 0 {
-				acks <- context.ReOpenDatabase()
+				err = context.ReOpenDatabase()
-			} else {
+			}
-				acks <- nil
+
+			request.err <- err
+
+			if err == nil {
+				clients++
 			}
-			clients++
 		case releasedb:
 			clients--
 			if clients == 0 {
 				flushColections()
-				acks <- context.CloseDatabase()
+				err = context.CloseDatabase()
 			} else {
-				acks <- nil
+				err = nil
 			}
+
+			request.err <- err
 		}
 	}
 }
@@ -107,7 +112,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 
 	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -115,7 +120,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 	if queryS != "" {
 		q, err := query.Parse(c.Request.URL.Query().Get("q"))
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 
@@ -132,7 +137,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 			sort.Strings(architecturesList)
 
 			if len(architecturesList) == 0 {
-				c.Fail(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
+				c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
 				return
 			}
 		}
@@ -142,7 +147,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList) {
 		list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
 			nil, context.DependencyOptions(), architecturesList)
 		if err != nil {
-			c.Fail(500, fmt.Errorf("unable to search: %s", err))
+			c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
 			return
 		}
 	}

api/files.go

@@ -24,7 +24,7 @@ func verifyPath(path string) bool {
 
 func verifyDir(c *gin.Context) bool {
 	if !verifyPath(c.Params.ByName("dir")) {
-		c.Fail(400, fmt.Errorf("wrong dir"))
+		c.AbortWithError(400, fmt.Errorf("wrong dir"))
 		return false
 	}
 
@@ -53,7 +53,7 @@ func apiFilesListDirs(c *gin.Context) {
 	})
 
 	if err != nil && !os.IsNotExist(err) {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -70,13 +70,13 @@ func apiFilesUpload(c *gin.Context) {
 	err := os.MkdirAll(path, 0777)
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -86,7 +86,7 @@ func apiFilesUpload(c *gin.Context) {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer src.Close()
@@ -94,14 +94,14 @@ func apiFilesUpload(c *gin.Context) {
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
 
@@ -138,9 +138,9 @@ func apiFilesListFiles(c *gin.Context) {
 
 	if err != nil {
 		if os.IsNotExist(err) {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 		} else {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 		}
 		return
 	}
@@ -156,7 +156,7 @@ func apiFilesDeleteDir(c *gin.Context) {
 
 	err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -170,14 +170,14 @@ func apiFilesDeleteFile(c *gin.Context) {
 	}
 
 	if !verifyPath(c.Params.ByName("name")) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
 	err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
 	if err != nil {
 		if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}

api/graph.go

@@ -53,25 +53,25 @@ func apiGraph(c *gin.Context) {
 
 	stdin, err := command.StdinPipe()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	_, err = io.Copy(stdin, buf)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = stdin.Close()
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	output, err = command.Output()
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
+		c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
 		return
 	}
 

api/packages.go

@@ -8,7 +8,7 @@ import (
 func apiPackagesShow(c *gin.Context) {
 	p, err := context.CollectionFactory().PackageCollection().ByKey([]byte(c.Params.ByName("key")))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 

api/publish.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -20,12 +21,12 @@ type SigningOptions struct {
 	PassphraseFile string
 }
 
-func getSigner(options *SigningOptions) (utils.Signer, error) {
+func getSigner(options *SigningOptions) (pgp.Signer, error) {
 	if options.Skip {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(options.GpgKey)
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
@@ -76,7 +77,7 @@ func apiPublishList(c *gin.Context) {
 	})
 
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -94,27 +95,29 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			Component string
 			Name      string `binding:"required"`
 		} `binding:"required"`
-		Distribution   string
+		Distribution         string
-		Label          string
+		Label                string
-		Origin         string
+		Origin               string
-		ForceOverwrite bool
+		NotAutomatic         string
-		SkipContents   *bool
+		ButAutomaticUpgrades string
-		Architectures  []string
+		ForceOverwrite       bool
-		Signing        SigningOptions
+		SkipContents         *bool
+		Architectures        []string
+		Signing              SigningOptions
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
 	if len(b.Sources) == 0 {
-		c.Fail(400, fmt.Errorf("unable to publish: soures are empty"))
+		c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
 		return
 	}
 
@@ -133,19 +136,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			snapshot, err = snapshotCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = snapshotCollection.LoadComplete(snapshot)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			sources = append(sources, snapshot)
 		}
-	} else if b.SourceKind == "local" {
+	} else if b.SourceKind == deb.SourceLocalRepo {
 		var localRepo *deb.LocalRepo
 
 		localCollection := context.CollectionFactory().LocalRepoCollection()
@@ -157,19 +160,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			localRepo, err = localCollection.ByName(source.Name)
 			if err != nil {
-				c.Fail(404, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
 				return
 			}
 
 			err = localCollection.LoadComplete(localRepo)
 			if err != nil {
-				c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+				c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 			}
 
 			sources = append(sources, localRepo)
 		}
 	} else {
-		c.Fail(400, fmt.Errorf("unknown SourceKind"))
+		c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
 		return
 	}
 
@@ -179,10 +182,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 	published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
-	published.Origin = b.Origin
+	if b.Origin != "" {
+		published.Origin = b.Origin
+	}
+	if b.NotAutomatic != "" {
+		published.NotAutomatic = b.NotAutomatic
+	}
+	if b.ButAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+	}
 	published.Label = b.Label
 
 	published.SkipContents = context.Config().SkipContentsPublishing
@@ -193,19 +204,19 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 	duplicate := collection.CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
-		c.Fail(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
+		c.AbortWithError(400, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate))
 		return
 	}
 
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
 
 	err = collection.Add(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
@@ -222,19 +233,21 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		ForceOverwrite bool
 		Signing        SigningOptions
 		SkipContents   *bool
+		SkipCleanup    *bool
 		Snapshots      []struct {
 			Component string `binding:"required"`
 			Name      string `binding:"required"`
 		}
+		AcquireByHash *bool
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
 	signer, err := getSigner(&b.Signing)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
 		return
 	}
 
@@ -253,20 +266,20 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
-		c.Fail(404, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 	err = collection.LoadComplete(published, context.CollectionFactory())
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
 	var updatedComponents []string
 
-	if published.SourceKind == "local" {
+	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
-			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
+			c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
 		updatedComponents = published.Components()
@@ -277,19 +290,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		publishedComponents := published.Components()
 		for _, snapshotInfo := range b.Snapshots {
 			if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
-				c.Fail(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
+				c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
 				return
 			}
 
-			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err != nil {
-				c.Fail(404, err)
+				c.AbortWithError(404, err2)
 				return
 			}
 
-			err = snapshotCollection.LoadComplete(snapshot)
+			err2 = snapshotCollection.LoadComplete(snapshot)
-			if err != nil {
+			if err2 != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err2)
 				return
 			}
 
@@ -297,7 +310,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			updatedComponents = append(updatedComponents, snapshotInfo.Component)
 		}
 	} else {
-		c.Fail(500, fmt.Errorf("unknown published repository type"))
+		c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
@@ -305,23 +318,29 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.SkipContents = *b.SkipContents
 	}
 
+	if b.AcquireByHash != nil {
+		published.AcquireByHash = *b.AcquireByHash
+	}
+
 	err = published.Publish(context.PackagePool(), context, context.CollectionFactory(), signer, nil, b.ForceOverwrite)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
 	err = collection.Update(published)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save to DB: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save to DB: %s", err))
 		return
 	}
 
-	err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
+	if b.SkipCleanup == nil || !*b.SkipCleanup {
-		context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
+		err = collection.CleanupPrefixComponentFiles(published.Prefix, updatedComponents,
-	if err != nil {
+			context.GetPublishedStorage(storage), context.CollectionFactory(), nil)
-		c.Fail(500, fmt.Errorf("unable to update: %s", err))
+		if err != nil {
-		return
+			c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
+			return
+		}
 	}
 
 	c.JSON(200, published)
@@ -330,6 +349,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 // DELETE /publish/:prefix/:distribution
 func apiPublishDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
+	skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
 
 	param := parseEscapedPath(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
@@ -345,9 +365,9 @@ func apiPublishDrop(c *gin.Context) {
 	defer collection.Unlock()
 
 	err := collection.Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), force)
+		context.CollectionFactory(), context.Progress(), force, skipCleanup)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to drop: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to drop: %s", err))
 		return
 	}
 

api/repos.go

@@ -37,7 +37,7 @@ func apiReposCreate(c *gin.Context) {
 		DefaultComponent    string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -51,7 +51,7 @@ func apiReposCreate(c *gin.Context) {
 
 	err := context.CollectionFactory().LocalRepoCollection().Add(repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -66,7 +66,7 @@ func apiReposEdit(c *gin.Context) {
 		DefaultComponent    *string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -76,7 +76,7 @@ func apiReposEdit(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -92,7 +92,7 @@ func apiReposEdit(c *gin.Context) {
 
 	err = collection.Update(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -107,7 +107,7 @@ func apiReposShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
@@ -132,27 +132,27 @@ func apiReposDrop(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.ByLocalRepo(repo)
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop, local repo is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop, local repo is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.ByLocalRepoSource(repo)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = collection.Drop(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -167,13 +167,13 @@ func apiReposPackagesShow(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -186,7 +186,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		PackageRefs []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -196,19 +196,19 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -219,15 +219,15 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = cb(list, p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -236,7 +236,7 @@ func apiReposPackagesAddDelete(c *gin.Context, cb func(list *deb.PackageList, p
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 
@@ -276,7 +276,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	fileParam := c.Params.ByName("file")
 	if fileParam != "" && !verifyPath(fileParam) {
-		c.Fail(400, fmt.Errorf("wrong file"))
+		c.AbortWithError(400, fmt.Errorf("wrong file"))
 		return
 	}
 
@@ -286,21 +286,22 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	repo, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
 	var (
 		sources                      []string
 		packageFiles, failedFiles    []string
+		otherFiles                   []string
 		processedFiles, failedFiles2 []string
 		reporter                     = &aptly.RecordingResultReporter{
 			Warnings:     []string{},
@@ -316,20 +317,22 @@ func apiReposPackageFromDir(c *gin.Context) {
 		sources = []string{filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("file"))}
 	}
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
 	list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), nil)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to load packages: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to load packages: %s", err))
 		return
 	}
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter, nil)
+		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to import package files: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to import package files: %s", err))
 		return
 	}
 
@@ -337,7 +340,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
 	if err != nil {
-		c.Fail(500, fmt.Errorf("unable to save: %s", err))
+		c.AbortWithError(500, fmt.Errorf("unable to save: %s", err))
 		return
 	}
 

api/router.go

@@ -20,32 +20,35 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
 		// running, we close the database to free the lock.
-		requests := make(chan int)
+		requests := make(chan dbRequest)
-		acks := make(chan error)
 
-		go acquireDatabase(requests, acks)
+		go acquireDatabase(requests)
-		go cacheFlusher(requests, acks)
 
 		router.Use(func(c *gin.Context) {
-			requests <- acquiredb
+			var err error
-			err := <-acks
+
+			errCh := make(chan error)
+			requests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
 			if err != nil {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 				return
 			}
+
 			defer func() {
-				requests <- releasedb
+				requests <- dbRequest{releasedb, errCh}
-				err = <-acks
+				err = <-errCh
 				if err != nil {
-					c.Fail(500, err)
+					c.AbortWithError(500, err)
-					return
 				}
 			}()
+
 			c.Next()
 		})
 
 	} else {
-		go cacheFlusher(nil, nil)
+		go cacheFlusher()
 	}
 
 	root := router.Group("/api")

api/snapshot.go

@@ -42,7 +42,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -56,25 +56,25 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = repo.CheckLock()
 	if err != nil {
-		c.Fail(409, err)
+		c.AbortWithError(409, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromRepository(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -84,7 +84,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -105,7 +105,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		PackageRefs     []string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -124,13 +124,13 @@ func apiSnapshotsCreate(c *gin.Context) {
 	for i := range b.SourceSnapshots {
 		sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
 		if err != nil {
-			c.Fail(404, err)
+			c.AbortWithError(404, err)
 			return
 		}
 
 		err = snapshotCollection.LoadComplete(sources[i])
 		if err != nil {
-			c.Fail(500, err)
+			c.AbortWithError(500, err)
 			return
 		}
 	}
@@ -144,15 +144,15 @@ func apiSnapshotsCreate(c *gin.Context) {
 		p, err = context.CollectionFactory().PackageCollection().ByKey([]byte(ref))
 		if err != nil {
 			if err == database.ErrNotFound {
-				c.Fail(404, fmt.Errorf("package %s: %s", ref, err))
+				c.AbortWithError(404, fmt.Errorf("package %s: %s", ref, err))
 			} else {
-				c.Fail(500, err)
+				c.AbortWithError(500, err)
 			}
 			return
 		}
 		err = list.Add(p)
 		if err != nil {
-			c.Fail(400, err)
+			c.AbortWithError(400, err)
 			return
 		}
 	}
@@ -161,7 +161,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -181,7 +181,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -195,19 +195,19 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	repo, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(repo)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	snapshot, err = deb.NewSnapshotFromLocalRepo(b.Name, repo)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -217,7 +217,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 
 	err = snapshotCollection.Add(snapshot)
 	if err != nil {
-		c.Fail(400, err)
+		c.AbortWithError(400, err)
 		return
 	}
 
@@ -236,7 +236,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		Description string
 	}
 
-	if !c.Bind(&b) {
+	if c.Bind(&b) != nil {
 		return
 	}
 
@@ -246,13 +246,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	snapshot, err = collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	_, err = collection.ByName(b.Name)
 	if err == nil {
-		c.Fail(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+		c.AbortWithError(409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
 		return
 	}
 
@@ -266,7 +266,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 
 	err = context.CollectionFactory().SnapshotCollection().Update(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -281,13 +281,13 @@ func apiSnapshotsShow(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -309,28 +309,28 @@ func apiSnapshotsDrop(c *gin.Context) {
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	published := publishedCollection.BySnapshot(snapshot)
 
 	if len(published) > 0 {
-		c.Fail(409, fmt.Errorf("unable to drop: snapshot is published"))
+		c.AbortWithError(409, fmt.Errorf("unable to drop: snapshot is published"))
 		return
 	}
 
 	if !force {
 		snapshots := snapshotCollection.BySnapshotSource(snapshot)
 		if len(snapshots) > 0 {
-			c.Fail(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
+			c.AbortWithError(409, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override"))
 			return
 		}
 	}
 
 	err = snapshotCollection.Drop(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -347,32 +347,32 @@ func apiSnapshotsDiff(c *gin.Context) {
 
 	snapshotA, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotA)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshotB)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
 	// Calculate diff
 	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), context.CollectionFactory().PackageCollection())
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 
@@ -397,13 +397,13 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 
 	snapshot, err := collection.ByName(c.Params.ByName("name"))
 	if err != nil {
-		c.Fail(404, err)
+		c.AbortWithError(404, err)
 		return
 	}
 
 	err = collection.LoadComplete(snapshot)
 	if err != nil {
-		c.Fail(500, err)
+		c.AbortWithError(500, err)
 		return
 	}
 

aptly/interfaces.go

@@ -3,25 +3,59 @@
 package aptly
 
 import (
+	"context"
 	"io"
+	"os"
 
 	"github.com/smira/aptly/utils"
 )
 
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
 // PackagePool is asbtraction of package pool storage.
 //
 // PackagePool stores all the package files, deduplicating them.
 type PackagePool interface {
-	// Path returns full path to package file in pool given any name and hash of file contents
+	// Verify checks whether file exists in the pool and fills back checksum info
-	Path(filename string, hashMD5 string) (string, error)
+	//
-	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
-	RelativePath(filename string, hashMD5 string) (string, error)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
 	FilepathList(progress Progress) ([]string, error)
 	// Remove deletes file in package pool returns its size
 	Remove(path string) (size int64, err error)
-	// Import copies file into package pool
+}
-	Import(path string, hashMD5 string) error
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
 }
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
@@ -35,15 +69,23 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
+	LinkFromPool(publishedDirectory, baseName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
+	// SymLink creates a symbolic link, which can be read with ReadLink
+	SymLink(src string, dst string) error
+	// HardLink creates a hardlink of a file
+	HardLink(src string, dst string) error
+	// FileExists returns true if path exists
+	FileExists(path string) (bool, error)
+	// ReadLink returns the symbolic link pointed to by path
+	ReadLink(path string) (string, error)
 }
 
-// LocalPublishedStorage is published storage on local filesystem
+// FileSystemPublishedStorage is published storage on filesystem
-type LocalPublishedStorage interface {
+type FileSystemPublishedStorage interface {
 	// PublicPath returns root of public part
 	PublicPath() string
 }
@@ -76,23 +118,24 @@ type Progress interface {
 	Printf(msg string, a ...interface{})
 	// ColoredPrintf does printf in colored way + newline
 	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
 }
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string, result chan<- error)
+	Download(ctx context.Context, url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
+	DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
-	// Pause pauses task processing
-	Pause()
-	// Resume resumes task processing
-	Resume()
-	// Shutdown stops downloader after current tasks are finished,
-	// but doesn't process rest of queue
-	Shutdown()
-	// Abort stops downloader without waiting for shutdown
-	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
+}

bash_completion.d/aptly

@@ -0,0 +1,633 @@
+#!/bin/bash
+
+# (The MIT License)
+#
+# Copyright (c) 2014 Andrey Smirnov
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+__aptly_mirror_list()
+{
+  aptly mirror list -raw
+}
+
+__aptly_repo_list()
+{
+  aptly repo list -raw
+}
+
+__aptly_snapshot_list()
+{
+  aptly snapshot list -raw
+}
+
+__aptly_published_distributions()
+{
+  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
+}
+
+__aptly_published_prefixes()
+{
+  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
+}
+
+__aptly_prefixes_for_distribution()
+{
+  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
+}
+
+_aptly()
+{
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+
+    commands="api config db graph mirror package publish repo serve snapshot task version"
+    options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
+    db_subcommands="cleanup recover"
+    mirror_subcommands="create drop edit show list rename search update"
+    publish_subcommands="drop list repo snapshot switch update"
+    snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
+    repo_subcommands="add copy create drop edit import include list move remove rename search show"
+    package_subcommands="search show"
+    task_subcommands="run"
+    config_subcommands="show"
+    api_subcommands="serve"
+
+    local cmd subcmd numargs numoptions i
+
+    numargs=0
+    numoptions=0
+
+    for (( i=1; i < $COMP_CWORD; i++ )); do
+      if [[ -n "$cmd" ]]; then
+        if [[ ! -n "$subcmd" ]]; then
+          subcmd=${COMP_WORDS[i]}
+          numargs=$(( COMP_CWORD - i - 1 ))
+        else
+          if [[ "${COMP_WORDS[i]}" == -* ]]; then
+            numoptions=$(( numoptions + 1 ))
+            numargs=$(( numargs - 1 ))
+          fi
+        fi
+      else
+        if [[ ! "${COMP_WORDS[i]}" == -* ]]; then
+          cmd=${COMP_WORDS[i]}
+        fi
+      fi
+    done
+
+    if [[ ! -n "$cmd" ]];
+    then
+      case "$cur" in
+            -*)
+              COMPREPLY=($(compgen -W "${options}" -- ${cur}))
+              return 0
+            ;;
+            *)
+              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
+              return 0
+            ;;
+        esac
+    fi
+
+    if [[ ! -n "$subcmd" ]];
+    then
+        case "$prev" in
+            "db")
+              COMPREPLY=($(compgen -W "${db_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "mirror")
+              COMPREPLY=($(compgen -W "${mirror_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "repo")
+              COMPREPLY=($(compgen -W "${repo_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "snapshot")
+              COMPREPLY=($(compgen -W "${snapshot_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "publish")
+              COMPREPLY=($(compgen -W "${publish_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "package")
+              COMPREPLY=($(compgen -W "${package_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "task")
+              COMPREPLY=($(compgen -W "${task_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "config")
+              COMPREPLY=($(compgen -W "${config_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "api")
+              COMPREPLY=($(compgen -W "${api_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            *)
+            ;;
+        esac
+    fi
+
+    case "$cmd" in
+      "mirror")
+        case "$subcmd" in
+          "create")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                return 0
+              fi
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "repo")
+        case "$subcmd" in
+          "add")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-force-replace -remove-files" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                _filedir '@(deb|dsc|udeb)'
+                return 0
+              ;;
+            esac
+          ;;
+          "copy"|"move")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "create")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "from" -- ${cur}))
+                return 0
+              ;;
+              2)
+                COMPREPLY=($(compgen -W "snapshot" -- ${cur}))
+                return 0
+              ;;
+              3)
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "include")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
+                else
+                  comptopt -o filenames 2>/dev/null
+                  COMPREPLY=($(compgen -f -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+            esac
+          ;;
+          "import")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "remove")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "snapshot")
+        case "$subcmd" in
+          "create")
+            case $numargs in
+              1)
+                COMPREPLY=($(compgen -W "from empty" -- ${cur}))
+                return 0
+              ;;
+              2)
+                if [[ "$prev" == "from" ]]; then
+                  COMPREPLY=($(compgen -W "mirror repo" -- ${cur}))
+                  return 0
+                fi
+              ;;
+              3)
+                if [[ "$prev" == "mirror" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                  return 0
+                fi
+                if [[ "$prev" == "repo" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                  return 0
+                fi
+              ;;
+            esac
+          ;;
+          "diff")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-only-matching" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw -sort=" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "merge")
+            if [[ $numargs -gt 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-latest" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "pull")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-all-matches -dry-run -no-deps -no-remove" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "filter")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "verify")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "publish")
+        case "$subcmd" in
+          "snapshot"|"repo")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+              else
+                if [[ "$subcmd" == "snapshot" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_published_prefixes)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "switch")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-cleanup -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -ge 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force-drop -skip-cleanup" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "package")
+        case "$subcmd" in
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-files -with-references" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "serve")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "graph")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-format= -output=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "api")
+        case "$subcmd" in
+          "serve")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "db")
+        case "$subcmd" in
+          "cleanup")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+    esac
+} && complete -F _aptly aptly

cmd/api_serve.go

@@ -59,11 +59,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
 		os.Remove(file)
-		listener, err := net.Listen("unix", file)
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
 		if err != nil {
 			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
 		}
 		defer listener.Close()
+
 		err = http.Serve(listener, api.Router(context))
 		if err != nil {
 			return fmt.Errorf("unable to serve: %s", err)

cmd/cmd.go

@@ -14,6 +14,12 @@ import (
 	"github.com/smira/flag"
 )
 
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
+)
+
 // ListPackagesRefList shows list of packages in PackageRefList
 func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	fmt.Printf("Packages:\n")
@@ -22,26 +28,22 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 		return
 	}
 
-	err = reflist.ForEach(func(key []byte) error {
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
-		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err2 != nil {
-			return err2
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	return
+	return PrintPackageList(list, "", "  ")
+
 }
 
 // PrintPackageList shows package list with specified format or default representation
-func PrintPackageList(result *deb.PackageList, format string) error {
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
 	if format == "" {
-		return result.ForEach(func(p *deb.Package) error {
+		return result.ForEachIndexed(func(p *deb.Package) error {
-			context.Progress().Printf("%s\n", p)
+			context.Progress().Printf(prefix+"%s\n", p)
 			return nil
 		})
 	}
@@ -51,13 +53,13 @@ func PrintPackageList(result *deb.PackageList, format string) error {
 		return fmt.Errorf("error parsing -format template: %s", err)
 	}
 
-	return result.ForEach(func(p *deb.Package) error {
+	return result.ForEachIndexed(func(p *deb.Package) error {
 		b := &bytes.Buffer{}
 		err = formatTemplate.Execute(b, p.ExtendedStanza())
 		if err != nil {
 			return fmt.Errorf("error applying template: %s", err)
 		}
-		context.Progress().Printf("%s\n", b.String())
+		context.Progress().Printf(prefix+"%s\n", b.String())
 		return nil
 	})
 
@@ -109,12 +111,15 @@ package environment to new version.`,
 		},
 	}
 
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/context.go

@@ -29,3 +29,8 @@ func InitContext(flags *flag.FlagSet) error {
 
 	return err
 }
+
+// GetContext gives access to the context
+func GetContext() *ctx.AptlyContext {
+	return context
+}

cmd/db_cleanup.go

@@ -37,9 +37,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
@@ -67,9 +67,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		if repo.RefList() != nil {
@@ -98,9 +98,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
@@ -125,12 +125,12 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
 		}
-		if published.SourceKind != "local" {
+		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		for _, component := range published.Components() {

cmd/mirror.go

@@ -3,19 +3,19 @@ package cmd
 import (
 	"strings"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
 	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}

cmd/mirror_edit.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -25,6 +26,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	fetchMirror := false
 	context.Flags().Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
@@ -35,6 +37,9 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.DownloadSources = flag.Value.Get().(bool)
 		case "with-udebs":
 			repo.DownloadUdebs = flag.Value.Get().(bool)
+		case "archive-url":
+			repo.SetArchiveRoot(flag.Value.String())
+			fetchMirror = true
 		}
 	})
 
@@ -51,8 +56,17 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 
 	if context.GlobalFlags().Lookup("architectures").Value.String() != "" {
 		repo.Architectures = context.ArchitecturesList()
+		fetchMirror = true
+	}
 
-		err = repo.Fetch(context.Downloader(), nil)
+	if fetchMirror {
+		var verifier pgp.Verifier
+		verifier, err = getVerifier(context.Flags())
+		if err != nil {
+			return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier)
 		if err != nil {
 			return fmt.Errorf("unable to edit: %s", err)
 		}
@@ -83,10 +97,13 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-edit", flag.ExitOnError),
 	}
 
+	cmd.Flag.String("archive-url", "", "archive url is the root of archive")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
 
 	return cmd
 }

cmd/mirror_show.go

@@ -37,21 +37,21 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	downloadSources := "no"
+	downloadSources := No
 	if repo.DownloadSources {
-		downloadSources = "yes"
+		downloadSources = Yes
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
-	downloadUdebs := "no"
+	downloadUdebs := No
 	if repo.DownloadUdebs {
-		downloadUdebs = "yes"
+		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
-		filterWithDeps := "no"
+		filterWithDeps := No
 		if repo.FilterWithDeps {
-			filterWithDeps = "yes"
+			filterWithDeps = Yes
 		}
 		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
 	}

cmd/mirror_update.go

@@ -2,10 +2,10 @@ package cmd
 
 import (
 	"fmt"
-	"os"
-	"os/signal"
 	"strings"
+	"sync"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/aptly/utils"
@@ -69,7 +69,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 
 		var oldLen, newLen int
-		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -81,8 +81,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		queue        []deb.PackageDownloadTask
 	)
 
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(),
+		context.CollectionFactory().ChecksumCollection(), skipExistingPackages)
+
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -107,9 +111,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	// Catch ^C
+	context.GoContextHandleSignals()
-	sigch := make(chan os.Signal)
-	signal.Notify(sigch, os.Interrupt)
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
@@ -117,48 +119,124 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// Download from the queue
 	context.Progress().InitBar(downloadSize, true)
 
-	// Download all package files
+	downloadQueue := make(chan int)
-	ch := make(chan error, count)
+
+	var (
+		errors  []string
+		errLock sync.Mutex
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
+	}
 
-	// In separate goroutine (to avoid blocking main), push queue to downloader
 	go func() {
-		for _, task := range queue {
+		for idx := range queue {
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
+			select {
+			case downloadQueue <- idx:
+			case <-context.Done():
+				return
+			}
 		}
-
+		close(downloadQueue)
-		// We don't need queue after this point
-		queue = nil
 	}()
 
-	// Wait for all downloads to finish
+	var wg sync.WaitGroup
-	var errors []string
 
-	for count > 0 {
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
-		select {
+		wg.Add(1)
-		case <-sigch:
+		go func() {
-			signal.Stop(sigch)
+			defer wg.Done()
-			return fmt.Errorf("unable to update: interrupted")
+			for {
-		case err = <-ch:
+				select {
-			if err != nil {
+				case idx, ok := <-downloadQueue:
-				errors = append(errors, err.Error())
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						context,
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreMismatch,
+						maxTries)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					task.Done = true
+				case <-context.Done():
+					return
+				}
 			}
-			count--
+		}()
-		}
 	}
 
+	// Wait for all download goroutines to finish
+	wg.Wait()
+
 	context.Progress().ShutdownBar()
-	signal.Stop(sigch)
-
-	if len(errors) > 0 {
-		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
-	}
 
 	err = context.ReOpenDatabase()
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	repo.FinalizeDownload()
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false)
+
+	for idx := range queue {
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		if !task.Done {
+			// download not finished yet
+			continue
+		}
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	select {
+	case <-context.Done():
+		return fmt.Errorf("unable to update: interrupted")
+	default:
+	}
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
+	}
+
+	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -188,6 +266,7 @@ Example:
 	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

cmd/package_search.go

@@ -35,7 +35,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format)
+	PrintPackageList(result, format, "")
 
 	return err
 }

cmd/package_show.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
@@ -13,9 +14,9 @@ import (
 
 func printReferencesTo(p *deb.Package) (err error) {
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -29,9 +30,9 @@ func printReferencesTo(p *deb.Package) (err error) {
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -45,20 +46,17 @@ func printReferencesTo(p *deb.Package) (err error) {
 	}
 
 	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if snapshot.RefList().Has(p) {
 			fmt.Printf("  snapshot %s\n", snapshot)
 		}
 		return nil
 	})
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }
 
 func aptlyPackageShow(cmd *commander.Command, args []string) error {
@@ -87,11 +85,18 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withFiles {
 			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
 			for _, f := range p.Files() {
-				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				var path string
+				path, err = f.GetPoolPath(packagePool)
 				if err != nil {
 					return err
 				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
 				fmt.Printf("  %s\n", path)
 			}
 			fmt.Printf("\n")

cmd/publish.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())

cmd/publish_drop.go

@@ -24,7 +24,9 @@ func aptlyPublishDrop(cmd *commander.Command, args []string) error {
 	storage, prefix := deb.ParsePrefix(param)
 
 	err = context.CollectionFactory().PublishedRepoCollection().Remove(context, storage, prefix, distribution,
-		context.CollectionFactory(), context.Progress(), context.Flags().Lookup("force-drop").Value.Get().(bool))
+		context.CollectionFactory(), context.Progress(),
+		context.Flags().Lookup("force-drop").Value.Get().(bool),
+		context.Flags().Lookup("skip-cleanup").Value.Get().(bool))
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -50,6 +52,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("force-drop", false, "remove published repository even if some files could not be cleaned up")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_list.go

@@ -20,9 +20,9 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		if raw {

cmd/publish_repo.go

@@ -43,8 +43,11 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_show.go

@@ -41,15 +41,15 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Sources:\n")
 	for component, sourceID := range repo.Sources {
 		var name string
-		if repo.SourceKind == "snapshot" {
+		if repo.SourceKind == deb.SourceSnapshot {
-			source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			source, e := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
-			if err != nil {
+			if e != nil {
 				continue
 			}
 			name = source.Name
-		} else if repo.SourceKind == "local" {
+		} else if repo.SourceKind == deb.SourceLocalRepo {
-			source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			source, e := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
-			if err != nil {
+			if e != nil {
 				continue
 			}
 			name = source.Name

cmd/publish_snapshot.go

@@ -35,7 +35,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		message string
 	)
 
-	if cmd.Name() == "snapshot" {
+	if cmd.Name() == "snapshot" { // nolint: goconst
 		var (
 			snapshot     *deb.Snapshot
 			emptyWarning = false
@@ -71,7 +71,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		if emptyWarning {
 			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
 		}
-	} else if cmd.Name() == "repo" {
+	} else if cmd.Name() == "repo" { // nolint: goconst
 		var (
 			localRepo    *deb.LocalRepo
 			emptyWarning = false
@@ -112,12 +112,23 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = context.Flags().Lookup("origin").Value.String()
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
 	published.Label = context.Flags().Lookup("label").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
@@ -126,6 +137,10 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.SkipContents = context.Flags().Lookup("skip-contents").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("acquire-by-hash") {
+		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
+	}
+
 	duplicate := context.CollectionFactory().PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
 		context.CollectionFactory().PublishedRepoCollection().LoadComplete(duplicate, context.CollectionFactory())
@@ -163,14 +178,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
 		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
 			localStorage.PublicPath())
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
 	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
 		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
@@ -211,9 +226,12 @@ Example:
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
-	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
 
 	return cmd
 }

cmd/publish_switch.go

@@ -44,7 +44,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "snapshot" {
+	if published.SourceKind != deb.SourceSnapshot {
 		return fmt.Errorf("unable to update: not a snapshot publish")
 	}
 
@@ -105,10 +105,13 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for snapshot %s has been successfully switched to new snapshot.\n", published.String())
@@ -151,6 +154,7 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/publish_update.go

@@ -30,7 +30,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "local" {
+	if published.SourceKind != deb.SourceLocalRepo {
 		return fmt.Errorf("unable to update: not a local repository publish")
 	}
 
@@ -69,10 +69,13 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
 
-	err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
+	skipCleanup := context.Flags().Lookup("skip-cleanup").Value.Get().(bool)
-		context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+	if !skipCleanup {
-	if err != nil {
+		err = context.CollectionFactory().PublishedRepoCollection().CleanupPrefixComponentFiles(published.Prefix, components,
-		return fmt.Errorf("unable to update: %s", err)
+			context.GetPublishedStorage(storage), context.CollectionFactory(), context.Progress())
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
 	}
 
 	context.Progress().Printf("\nPublish for local repo %s has been successfully updated.\n", published.String())
@@ -109,6 +112,7 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 
 	return cmd
 }

cmd/repo_add.go

@@ -20,7 +20,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
 	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
@@ -41,19 +41,22 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
 
-	var packageFiles, failedFiles []string
+	var packageFiles, otherFiles, failedFiles []string
 
-	packageFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
+	packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(args[1:], &aptly.ConsoleResultReporter{Progress: context.Progress()})
 
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)
 	}
 
+	processedFiles = append(processedFiles, otherFiles...)
+
 	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
 	err = context.CollectionFactory().LocalRepoCollection().Update(repo)
@@ -65,7 +68,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}

cmd/repo_create.go

@@ -10,7 +10,7 @@ import (
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
 		cmd.Usage()
 		return commander.ErrCommandError
 	}

cmd/repo_include.go

@@ -28,7 +28,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	}
 
 	if verifier == nil {
-		verifier = &utils.GpgVerifier{}
+		verifier = context.GetVerifier()
 	}
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
@@ -97,7 +97,8 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
 
-		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		var repo *deb.LocalRepo
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
@@ -131,12 +132,13 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load repo: %s", err)
 		}
 
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		var list *deb.PackageList
+		list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to load packages: %s", err)
 		}
 
-		packageFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
+		packageFiles, otherFiles, _ := deb.CollectPackageFiles([]string{changes.TempDir}, reporter)
 
 		var restriction deb.PackageQuery
 
@@ -151,7 +153,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 		var processedFiles2, failedFiles2 []string
 
 		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			context.CollectionFactory().PackageCollection(), reporter, restriction)
+			context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())
 
 		if err != nil {
 			return fmt.Errorf("unable to import package files: %s", err)
@@ -177,6 +179,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
 		}
 
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
 		processedFiles = append(processedFiles, path)
 	}
 
@@ -184,7 +190,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}

cmd/repo_list.go

@@ -23,9 +23,9 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+			e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-			if err != nil {
+			if e != nil {
-				return err
+				return e
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())

cmd/repo_move.go

@@ -34,7 +34,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		srcRepo    *deb.LocalRepo
 	)
 
-	if command == "copy" || command == "move" {
+	if command == "copy" || command == "move" { // nolint: goconst
 		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
@@ -50,7 +50,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 
 		srcRefList = srcRepo.RefList()
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
 		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
@@ -115,18 +115,18 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var verb string
 
-	if command == "move" {
+	if command == "move" { // nolint: goconst
 		verb = "moved"
-	} else if command == "copy" {
+	} else if command == "copy" { // nolint: goconst
 		verb = "copied"
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		verb = "imported"
 	}
 
@@ -136,7 +136,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcList.Remove(p)
 		}
 		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
@@ -156,7 +156,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
 			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)

cmd/serve.go

@@ -60,9 +60,9 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		sources = append(sources, repo.String())
@@ -90,13 +90,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
 			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
-		if utils.StrSliceHasItem(repo.Architectures, "source") {
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
 				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot_create.go

@@ -13,7 +13,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
@@ -38,7 +38,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
-	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
 		// aptly snapshot create snap from repo repo
 		var repo *deb.LocalRepo
 

cmd/snapshot_filter.go

@@ -66,7 +66,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}

cmd/snapshot_pull.go

@@ -96,7 +96,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}

cmd/snapshot_search.go

@@ -26,8 +26,9 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	var reflist *deb.PackageRefList
 
-	if command == "snapshot" {
+	if command == "snapshot" { // nolint: goconst
-		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		var snapshot *deb.Snapshot
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -39,7 +40,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 		reflist = snapshot.RefList()
 	} else if command == "mirror" {
-		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		var repo *deb.RemoteRepo
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -50,8 +52,9 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 
 		reflist = repo.RefList()
-	} else if command == "repo" {
+	} else if command == "repo" { // nolint: goconst
-		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		var repo *deb.LocalRepo
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -99,8 +102,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 	}
 
-	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList)
+		nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -110,7 +113,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format)
+	PrintPackageList(result, format, "")
 
 	return err
 }

cmd/snapshot_show.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -34,20 +35,23 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("Sources:\n")
 		for _, sourceID := range snapshot.SourceIDs {
 			var name string
-			if snapshot.SourceKind == "snapshot" {
+			if snapshot.SourceKind == deb.SourceSnapshot {
-				source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				var source *deb.Snapshot
+				source, err = context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
-			} else if snapshot.SourceKind == "local" {
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
-				source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				var source *deb.LocalRepo
+				source, err = context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
-			} else if snapshot.SourceKind == "repo" {
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
-				source, err := context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				var source *deb.RemoteRepo
+				source, err = context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}

cmd/task_run.go

@@ -82,7 +82,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 	for i, command := range cmdList {
 		if !commandErrored {
-			err := context.ReOpenDatabase()
+			err = context.ReOpenDatabase()
 			if err != nil {
 				return fmt.Errorf("failed to reopen DB: %s", err)
 			}

console/progress.go

@@ -2,6 +2,7 @@ package console
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/cheggaaa/pb"
@@ -11,6 +12,7 @@ import (
 
 const (
 	codePrint = iota
+	codePrintStdErr
 	codeProgress
 	codeHideProgress
 	codeStop
@@ -28,7 +30,6 @@ type printTask struct {
 // Progress is a progress displaying subroutine, it allows to show download and other operations progress
 // mixed with progress bar
 type Progress struct {
-	stop     chan bool
 	stopped  chan bool
 	queue    chan printTask
 	bar      *pb.ProgressBar
@@ -128,6 +129,11 @@ func (p *Progress) Printf(msg string, a ...interface{}) {
 	p.queue <- printTask{code: codePrint, message: fmt.Sprintf(msg, a...)}
 }
 
+// PrintfStdErr does printf but in safe manner to stderr
+func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
+	p.queue <- printTask{code: codePrintStdErr, message: fmt.Sprintf(msg, a...)}
+}
+
 // ColoredPrintf does printf in colored way + newline
 func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 	if RunningOnTerminal() {
@@ -183,6 +189,12 @@ func (p *Progress) worker() {
 				p.barShown = false
 			}
 			fmt.Print(task.message)
+		case codePrintStdErr:
+			if p.barShown {
+				fmt.Print("\r\033[2K")
+				p.barShown = false
+			}
+			fmt.Fprint(os.Stderr, task.message)
 		case codeProgress:
 			if hasBar {
 				fmt.Print("\r" + task.message)

context/context.go

@@ -2,8 +2,11 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
+	"math/rand"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
 	"runtime/pprof"
@@ -17,6 +20,7 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/swift"
 	"github.com/smira/aptly/utils"
@@ -28,6 +32,8 @@ import (
 type AptlyContext struct {
 	sync.Mutex
 
+	gocontext.Context
+
 	flags, globalFlags *flag.FlagSet
 	configLoaded       bool
 
@@ -101,6 +107,9 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+
+				// as this is fresh aptly installation, we don't need to support legacy pool locations
+				utils.Config.SkipLegacyPool = true
 				utils.SaveConfig(configLocations[0], &utils.Config)
 			}
 		}
@@ -149,6 +158,9 @@ func (context *AptlyContext) DependencyOptions() int {
 		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
 			context.dependencyOptions |= deb.DepFollowSource
 		}
+		if context.lookupOption(context.config().DepVerboseResolve, "dep-verbose-resolve") {
+			context.dependencyOptions |= deb.DepVerboseResolve
+		}
 	}
 
 	return context.dependencyOptions
@@ -201,8 +213,7 @@ func (context *AptlyContext) Downloader() aptly.Downloader {
 		if downloadLimit == 0 {
 			downloadLimit = context.config().DownloadLimit
 		}
-		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+		context.downloader = http.NewDownloader(downloadLimit*1024, context._progress())
-			downloadLimit*1024, context._progress())
 	}
 
 	return context.downloader
@@ -233,13 +244,34 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 	if context.database == nil {
 		var err error
 
-		context.database, err = database.OpenDB(context.dbPath())
+		context.database, err = database.NewDB(context.dbPath())
 		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
+			return nil, fmt.Errorf("can't instantiate database: %s", err)
 		}
 	}
 
-	return context.database, nil
+	tries := context.flags.Lookup("db-open-attempts").Value.Get().(int)
+	const BaseDelay = 10 * time.Second
+	const Jitter = 1 * time.Second
+
+	for ; tries >= 0; tries-- {
+		err := context.database.Open()
+		if err == nil || !strings.Contains(err.Error(), "resource temporarily unavailable") {
+			return context.database, err
+		}
+
+		if tries > 0 {
+			delay := time.Duration(rand.NormFloat64()*float64(Jitter) + float64(BaseDelay))
+			if delay < 0 {
+				delay = time.Second
+			}
+
+			context._progress().PrintfStdErr("Unable to open database, sleeping %s, attempts left %d...\n", delay, tries)
+			time.Sleep(delay)
+		}
+	}
+
+	return nil, fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
 }
 
 // CloseDatabase closes the db temporarily
@@ -256,26 +288,9 @@ func (context *AptlyContext) CloseDatabase() error {
 
 // ReOpenDatabase reopens the db after close
 func (context *AptlyContext) ReOpenDatabase() error {
-	context.Lock()
+	_, err := context.Database()
-	defer context.Unlock()
 
-	if context.database == nil {
+	return err
-		return nil
-	}
-
-	const MaxTries = 10
-	const Delay = 10 * time.Second
-
-	for try := 0; try < MaxTries; try++ {
-		err := context.database.ReOpen()
-		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
-			return err
-		}
-		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
-		<-time.After(Delay)
-	}
-
-	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
 }
 
 // CollectionFactory builds factory producing all kinds of collections
@@ -300,7 +315,7 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	defer context.Unlock()
 
 	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.config().RootDir)
+		context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
 	}
 
 	return context.packagePool
@@ -314,7 +329,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 	publishedStorage, ok := context.publishedStorages[name]
 	if !ok {
 		if name == "" {
-			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
+		} else if strings.HasPrefix(name, "filesystem:") {
+			params, ok := context.config().FileSystemPublishRoots[name[11:]]
+			if !ok {
+				Fatal(fmt.Errorf("published local storage %v not configured", name[6:]))
+			}
+
+			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
 		} else if strings.HasPrefix(name, "s3:") {
 			params, ok := context.config().S3PublishRoots[name[3:]]
 			if !ok {
@@ -356,6 +378,46 @@ func (context *AptlyContext) UploadPath() string {
 	return filepath.Join(context.Config().RootDir, "upload")
 }
 
+func (context *AptlyContext) pgpProvider() string {
+	var provider string
+
+	if context.globalFlags.IsSet("gpg-provider") {
+		provider = context.globalFlags.Lookup("gpg-provider").Value.String()
+	} else {
+		provider = context.config().GpgProvider
+	}
+
+	if !(provider == "gpg" || provider == "internal") { // nolint: goconst
+		Fatal(fmt.Errorf("unknown gpg provider: %v", provider))
+	}
+
+	return provider
+}
+
+// GetSigner returns Signer with respect to provider
+func (context *AptlyContext) GetSigner() pgp.Signer {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.pgpProvider() == "gpg" { // nolint: goconst
+		return &pgp.GpgSigner{}
+	}
+
+	return &pgp.GoSigner{}
+}
+
+// GetVerifier returns Verifier with respect to provider
+func (context *AptlyContext) GetVerifier() pgp.Verifier {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.pgpProvider() == "gpg" { // nolint: goconst
+		return &pgp.GpgVerifier{}
+	}
+
+	return &pgp.GoVerifier{}
+}
+
 // UpdateFlags sets internal copy of flags in the context
 func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
 	context.Lock()
@@ -380,6 +442,27 @@ func (context *AptlyContext) GlobalFlags() *flag.FlagSet {
 	return context.globalFlags
 }
 
+// GoContextHandleSignals upgrades context to handle ^C by aborting context
+func (context *AptlyContext) GoContextHandleSignals() {
+	context.Lock()
+	defer context.Unlock()
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	var cancel gocontext.CancelFunc
+
+	context.Context, cancel = gocontext.WithCancel(context.Context)
+
+	go func() {
+		<-sigch
+		signal.Stop(sigch)
+		context.Progress().PrintfStdErr("Aborting... press ^C once again to abort immediately\n")
+		cancel()
+	}()
+}
+
 // Shutdown shuts context down
 func (context *AptlyContext) Shutdown() {
 	context.Lock()
@@ -406,7 +489,6 @@ func (context *AptlyContext) Shutdown() {
 		context.database = nil
 	}
 	if context.downloader != nil {
-		context.downloader.Abort()
 		context.downloader = nil
 	}
 	if context.progress != nil {
@@ -421,7 +503,6 @@ func (context *AptlyContext) Cleanup() {
 	defer context.Unlock()
 
 	if context.downloader != nil {
-		context.downloader.Shutdown()
 		context.downloader = nil
 	}
 	if context.progress != nil {
@@ -438,6 +519,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 		flags:             flags,
 		globalFlags:       flags,
 		dependencyOptions: -1,
+		Context:           gocontext.TODO(),
 		publishedStorages: map[string]aptly.PublishedStorage{},
 	}
 

database/leveldb.go

@@ -32,8 +32,8 @@ type Storage interface {
 	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
+	Open() error
 	Close() error
-	ReOpen() error
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
@@ -66,13 +66,19 @@ func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	return leveldb.OpenFile(path, o)
 }
 
-// OpenDB opens (creates) LevelDB database
+// NewDB creates new instance of DB, but doesn't open it (yet)
-func OpenDB(path string) (Storage, error) {
+func NewDB(path string) (Storage, error) {
-	db, err := internalOpen(path, false)
+	return &levelDB{path: path}, nil
+}
+
+// NewOpenDB creates new instance of DB and opens it
+func NewOpenDB(path string) (Storage, error) {
+	db, err := NewDB(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db, path: path}, nil
+
+	return db, db.Open()
 }
 
 // RecoverDB recovers LevelDB database from corruption
@@ -215,8 +221,8 @@ func (l *levelDB) Close() error {
 	return err
 }
 
-// Reopen tries to re-open the database
+// Reopen tries to open (re-open) the database
-func (l *levelDB) ReOpen() error {
+func (l *levelDB) Open() error {
 	if l.db != nil {
 		return nil
 	}

database/leveldb_test.go

@@ -22,7 +22,7 @@ func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
 	s.path = c.MkDir()
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -46,7 +46,7 @@ func (s *LevelDBSuite) TestRecoverDB(c *C) {
 	err = RecoverDB(s.path)
 	c.Check(err, IsNil)
 
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Check(err, IsNil)
 
 	result, err := s.db.Get(key)
@@ -223,7 +223,7 @@ func (s *LevelDBSuite) TestReOpen(c *C) {
 	err = s.db.Close()
 	c.Assert(err, IsNil)
 
-	err = s.db.ReOpen()
+	err = s.db.Open()
 	c.Assert(err, IsNil)
 
 	result, err := s.db.Get(key)

deb/changes.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -23,7 +25,7 @@ type Changes struct {
 	Binary                []string
 	Architectures         []string
 	Stanza                Stanza
-	SignatureKeys         []utils.GpgKey
+	SignatureKeys         []pgp.Key
 }
 
 // NewChanges moves .changes file into temporary directory and creates Changes structure
@@ -50,7 +52,7 @@ func NewChanges(path string) (*Changes, error) {
 }
 
 // VerifyAndParse does optional signature verification and parses changes files
-func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier pgp.Verifier) error {
 	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
 	if err != nil {
 		return err
@@ -69,7 +71,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	}
 
 	if isClearSigned && !ignoreSignature {
-		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		var keyInfo *pgp.KeyInfo
+		keyInfo, err = verifier.VerifyClearsigned(input, false)
 		if err != nil {
 			return err
 		}
@@ -78,7 +81,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		c.SignatureKeys = keyInfo.GoodKeys
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(input)
@@ -103,11 +106,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	c.Architectures = strings.Fields(c.Stanza["Architecture"])
 
 	c.Files, err = c.Files.ParseSumFields(c.Stanza)
-	if err != nil {
+	return err
-		return err
-	}
-
-	return nil
 }
 
 // Prepare creates temporary directory, copies file there and verifies checksums
@@ -173,7 +172,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 
 	// if c.Source is empty, this would never match
 	sourceQuery := &AndQuery{
-		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource},
 		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
 	}
 
@@ -181,8 +180,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 	if len(c.Binary) > 0 {
 		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
 		// matching debug ddeb packages, they're not present in the Binary field
-		var ddebQuery PackageQuery
+		var ddebQuery PackageQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
-		ddebQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
 
 		for _, binary := range c.Binary[1:] {
 			binaryQuery = &OrQuery{
@@ -206,7 +204,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		}
 
 		binaryQuery = &AndQuery{
-			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource}},
 			R: binaryQuery}
 	}
 

deb/checksum_collection.go

@@ -0,0 +1,67 @@
+package deb
+
+import (
+	"bytes"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+	"github.com/ugorji/go/codec"
+)
+
+// ChecksumCollection does management of ChecksumInfo in DB
+type ChecksumCollection struct {
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
+}
+
+// NewChecksumCollection creates new ChecksumCollection and binds it to database
+func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+	return &ChecksumCollection{
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
+	}
+}
+
+func (collection *ChecksumCollection) dbKey(path string) []byte {
+	return []byte("C" + path)
+}
+
+// Get finds checksums in DB by path
+func (collection *ChecksumCollection) Get(path string) (*utils.ChecksumInfo, error) {
+	encoded, err := collection.db.Get(collection.dbKey(path))
+	if err != nil {
+		if err == database.ErrNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	c := &utils.ChecksumInfo{}
+
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+	err = decoder.Decode(c)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Update adds or updates information about checksum in DB
+func (collection *ChecksumCollection) Update(path string, c *utils.ChecksumInfo) error {
+	var encodeBuffer bytes.Buffer
+
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+	err := encoder.Encode(c)
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Put(collection.dbKey(path), encodeBuffer.Bytes())
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &ChecksumCollection{}
+)

deb/checksum_collection_test.go

@@ -0,0 +1,47 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type ChecksumCollectionSuite struct {
+	collection *ChecksumCollection
+	c          utils.ChecksumInfo
+	db         database.Storage
+}
+
+var _ = Suite(&ChecksumCollectionSuite{})
+
+func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
+	s.c = utils.ChecksumInfo{
+		Size:   124,
+		MD5:    "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+	}
+	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.collection = NewChecksumCollection(s.db)
+}
+
+func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *ChecksumCollectionSuite) TestFlow(c *C) {
+	// checksum not stored
+	checksum, err := s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(checksum, IsNil)
+
+	// store checksum
+	err = s.collection.Update("some/path", &s.c)
+	c.Assert(err, IsNil)
+
+	// load it back
+	checksum, err = s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(*checksum, DeepEquals, s.c)
+}

deb/collections.go

@@ -15,6 +15,7 @@ type CollectionFactory struct {
 	snapshots      *SnapshotCollection
 	localRepos     *LocalRepoCollection
 	publishedRepos *PublishedRepoCollection
+	checksums      *ChecksumCollection
 }
 
 // NewCollectionFactory creates new factory
@@ -89,6 +90,18 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 	return factory.publishedRepos
 }
 
+// ChecksumCollection returns (or creates) new ChecksumCollection
+func (factory *CollectionFactory) ChecksumCollection() *ChecksumCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.checksums == nil {
+		factory.checksums = NewChecksumCollection(factory.db)
+	}
+
+	return factory.checksums
+}
+
 // Flush removes all references to collections, so that memory could be reclaimed
 func (factory *CollectionFactory) Flush() {
 	factory.Lock()
@@ -99,4 +112,5 @@ func (factory *CollectionFactory) Flush() {
 	factory.remoteRepos = nil
 	factory.publishedRepos = nil
 	factory.packages = nil
+	factory.checksums = nil
 }

deb/contents.go

@@ -26,8 +26,8 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) error {
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
-	contents := p.Contents(packagePool)
+	contents := p.Contents(packagePool, progress)
 	qualifiedName := []byte(p.QualifiedName())
 
 	for _, path := range contents {

deb/deb.go

@@ -12,12 +12,20 @@ import (
 
 	"github.com/h2non/filetype/matchers"
 	"github.com/mkrautz/goar"
+	"github.com/pkg/errors"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/go-xz"
 	"github.com/smira/lzma"
 )
 
+// Source kinds
+const (
+	SourceSnapshot   = "snapshot"
+	SourceLocalRepo  = "local"
+	SourceRemoteRepo = "repo"
+)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -29,21 +37,46 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
+
 		if err == io.EOF {
-			return nil, fmt.Errorf("unable to find control.tar.gz part in package %s", packageFile)
+			return nil, fmt.Errorf("unable to find control.tar.* part in package %s", packageFile)
 		}
 		if err != nil {
 			return nil, fmt.Errorf("unable to read .deb archive %s: %s", packageFile, err)
 		}
 
-		if header.Name == "control.tar.gz" {
+		// As per deb(5) version 1.19.0.4 the control file may be:
-			ungzip, err := gzip.NewReader(library)
+		// - control.tar (since 1.17.6)
-			if err != nil {
+		// - control.tar.gz
-				return nil, fmt.Errorf("unable to ungzip control file from %s. Error: %s", packageFile, err)
+		// - control.tar.xz (since 1.17.6)
-			}
+		// Look for all of the above and uncompress as necessary.
-			defer ungzip.Close()
+		if strings.HasPrefix(header.Name, "control.tar") {
+			bufReader := bufio.NewReader(library)
 
-			untar := tar.NewReader(ungzip)
+			var tarInput io.Reader
+
+			switch header.Name {
+			case "control.tar":
+				tarInput = bufReader
+			case "control.tar.gz":
+				ungzip, err := gzip.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
+				}
+				defer ungzip.Close()
+				tarInput = ungzip
+			case "control.tar.xz":
+				unxz, err := xz.NewReader(bufReader)
+				if err != nil {
+					return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
+				}
+				defer unxz.Close()
+				tarInput = unxz
+			default:
+				return nil, fmt.Errorf("unsupported tar compression in %s: %s", packageFile, header.Name)
+			}
+
+			untar := tar.NewReader(tarInput)
 			for {
 				tarHeader, err := untar.Next()
 				if err == io.EOF {
@@ -68,7 +101,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 }
 
 // GetControlFileFromDsc reads control file from dsc package
-func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, error) {
+func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error) {
 	file, err := os.Open(dscFile)
 	if err != nil {
 		return nil, err
@@ -82,7 +115,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 		return nil, err
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
@@ -105,13 +138,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 }
 
 // GetContentsFromDeb returns list of files installed by .deb package
-func GetContentsFromDeb(packageFile string) ([]string, error) {
+func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
-	file, err := os.Open(packageFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
@@ -119,7 +146,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+			return nil, errors.Wrapf(err, "unable to read .deb archive from %s", packageFile)
 		}
 
 		if strings.HasPrefix(header.Name, "data.tar") {
@@ -142,7 +169,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 				} else {
 					ungzip, err := gzip.NewReader(bufReader)
 					if err != nil {
-						return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+						return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
 					}
 					defer ungzip.Close()
 					tarInput = ungzip
@@ -152,7 +179,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			case "data.tar.xz":
 				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
-					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
 				}
 				defer unxz.Close()
 				tarInput = unxz
@@ -172,16 +199,14 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 					return results, nil
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to read .tar archive from %s", packageFile)
 				}
 
 				if tarHeader.Typeflag == tar.TypeDir {
 					continue
 				}
 
-				if strings.HasPrefix(tarHeader.Name, "./") {
+				tarHeader.Name = strings.TrimPrefix(tarHeader.Name[2:], "./")
-					tarHeader.Name = tarHeader.Name[2:]
-				}
 				results = append(results, tarHeader.Name)
 			}
 		}

deb/deb_test.go

@@ -1,16 +1,17 @@
 package deb
 
 import (
+	"os"
 	"path/filepath"
 	"runtime"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 
 	. "gopkg.in/check.v1"
 )
 
 type DebSuite struct {
-	debFile, debFile2, dscFile, dscFileNoSign string
+	debFile, debFile2, debFileWithXzControl, dscFile, dscFileNoSign string
 }
 
 var _ = Suite(&DebSuite{})
@@ -19,6 +20,7 @@ func (s *DebSuite) SetUpSuite(c *C) {
 	_, _File, _, _ := runtime.Caller(0)
 	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
 	s.debFile2 = filepath.Join(filepath.Dir(_File), "../system/changes/hardlink_0.2.1_amd64.deb")
+	s.debFileWithXzControl = filepath.Join(filepath.Dir(_File), "../system/changes/libqt5concurrent5-dbgsym_5.9.1+dfsg-2+18.04+bionic+build4_amd64.ddeb")
 	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
 	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
 }
@@ -37,8 +39,16 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 	c.Check(st["Package"], Equals, "libboost-program-options-dev")
 }
 
+func (s *DebSuite) TestGetControlFileFromDebWithXzControl(c *C) {
+	// Has control.tar.xz archive inside.
+	st, err := GetControlFileFromDeb(s.debFileWithXzControl)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "5.9.1+dfsg-2+18.04+bionic+build4")
+	c.Check(st["Package"], Equals, "libqt5concurrent5-dbgsym")
+}
+
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
-	verifier := &utils.GpgVerifier{}
+	verifier := &pgp.GoVerifier{}
 
 	_, err := GetControlFileFromDsc("/no/such/file", verifier)
 	c.Check(err, ErrorMatches, ".*no such file or directory")
@@ -59,13 +69,19 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 }
 
 func (s *DebSuite) TestGetContentsFromDeb(c *C) {
-	contents, err := GetContentsFromDeb(s.debFile)
+	f, err := os.Open(s.debFile)
+	c.Assert(err, IsNil)
+	contents, err := GetContentsFromDeb(f, s.debFile)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
 		"usr/share/doc/libboost-program-options-dev/copyright"})
+	c.Assert(f.Close(), IsNil)
 
-	contents, err = GetContentsFromDeb(s.debFile2)
+	f, err = os.Open(s.debFile2)
+	c.Assert(err, IsNil)
+	contents, err = GetContentsFromDeb(f, s.debFile2)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
 		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+	c.Assert(f.Close(), IsNil)
 }

deb/format.go

@@ -22,6 +22,8 @@ var (
 		"Version",
 		"Codename",
 		"Date",
+		"NotAutomatic",
+		"ButAutomaticUpgrades",
 		"Architectures",
 		"Architecture",
 		"Components",

deb/graph.go

@@ -33,9 +33,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
-		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -55,9 +55,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
-		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -81,13 +81,13 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	})
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
-		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
+		if snapshot.SourceKind == SourceRemoteRepo {
 			description = "Snapshot from repo"
 		}
 
@@ -99,7 +99,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 				snapshot.Name, description, snapshot.NumPackages(), labelEnd),
 		})
 
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+		if snapshot.SourceKind == SourceRemoteRepo || snapshot.SourceKind == SourceLocalRepo || snapshot.SourceKind == SourceSnapshot {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {

deb/import.go

@@ -7,11 +7,12 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
 // CollectPackageFiles walks filesystem collecting all candidates for package files
-func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, failedFiles []string) {
+func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
 	for _, location := range locations {
 		info, err2 := os.Stat(location)
 		if err2 != nil {
@@ -31,6 +32,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 					strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 					packageFiles = append(packageFiles, path)
+				} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+					otherFiles = append(otherFiles, path)
 				}
 
 				return nil
@@ -45,6 +48,8 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
 				strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
 				packageFiles = append(packageFiles, location)
+			} else if strings.HasSuffix(info.Name(), ".buildinfo") {
+				otherFiles = append(otherFiles, location)
 			} else {
 				reporter.Warning("Unknown file extension: %s", location)
 				failedFiles = append(failedFiles, location)
@@ -59,8 +64,9 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 }
 
 // ImportPackageFiles imports files into local repository
-func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier pgp.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery,
+	checksumStorage aptly.ChecksumStorage) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -116,19 +122,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		var files PackageFiles
+
+		if isSourcePackage {
+			files = p.Files()
+		}
+
 		var checksums utils.ChecksumInfo
 		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		if isSourcePackage {
+		mainPackageFile := PackageFile{
-			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+			Filename:  filepath.Base(file),
-		} else {
+			Checksums: checksums,
-			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
 		}
 
-		err = pool.Import(file, checksums.MD5)
+		mainPackageFile.PoolPath, err = pool.Import(file, mainPackageFile.Filename, &mainPackageFile.Checksums, false, checksumStorage)
 		if err != nil {
 			reporter.Warning("Unable to import file %s into pool: %s", file, err)
 			failedFiles = append(failedFiles, file)
@@ -137,26 +148,45 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 
 		candidateProcessedFiles = append(candidateProcessedFiles, file)
 
-		// go over all files, except for the last one (.dsc/.deb itself)
+		// go over all the other files
-		for _, f := range p.Files() {
+		for i := range files {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-				continue
+
+			_, err = os.Stat(sourceFile)
+			if err == nil {
+				files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
+				if err == nil {
+					candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+				}
+			} else if os.IsNotExist(err) {
+				// if file is not present, try to find it in the pool
+				var (
+					err2  error
+					found bool
+				)
+
+				files[i].PoolPath, found, err2 = pool.Verify("", files[i].Filename, &files[i].Checksums, checksumStorage)
+				if err2 != nil {
+					err = err2
+				} else if found {
+					// clear error, file is already in the package pool
+					err = nil
+				}
 			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
+
-			err = pool.Import(sourceFile, f.Checksums.MD5)
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
 				break
 			}
-
-			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
 		}
 		if err != nil {
 			// some files haven't been imported
 			continue
 		}
 
+		p.UpdateFiles(append(files, mainPackageFile))
+
 		if restriction != nil && !restriction.Matches(p) {
 			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
 			failedFiles = append(failedFiles, file)

deb/index_files.go

@@ -4,10 +4,12 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -19,18 +21,20 @@ type indexFiles struct {
 	tempDir          string
 	suffix           string
 	indexes          map[string]*indexFile
+	acquireByHash    bool
 }
 
 type indexFile struct {
-	parent       *indexFiles
+	parent        *indexFiles
-	discardable  bool
+	discardable   bool
-	compressable bool
+	compressable  bool
-	onlyGzip     bool
+	onlyGzip      bool
-	signable     bool
+	signable      bool
-	relativePath string
+	acquireByHash bool
-	tempFilename string
+	relativePath  string
-	tempFile     *os.File
+	tempFilename  string
-	w            *bufio.Writer
+	tempFile      *os.File
+	w             *bufio.Writer
 }
 
 func (file *indexFile) BufWriter() (*bufio.Writer, error) {
@@ -48,7 +52,7 @@ func (file *indexFile) BufWriter() (*bufio.Writer, error) {
 	return file.w, nil
 }
 
-func (file *indexFile) Finalize(signer utils.Signer) error {
+func (file *indexFile) Finalize(signer pgp.Signer) error {
 	if file.w == nil {
 		if file.discardable {
 			return nil
@@ -90,11 +94,22 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
 	}
 
-	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+
+	err = file.parent.publishedStorage.MkDir(filedir)
 	if err != nil {
 		return fmt.Errorf("unable to create dir: %s", err)
 	}
 
+	if file.acquireByHash {
+		for _, hash := range []string{"MD5Sum", "SHA1", "SHA256", "SHA512"} {
+			err = file.parent.publishedStorage.MkDir(filepath.Join(filedir, "by-hash", hash))
+			if err != nil {
+				return fmt.Errorf("unable to create dir: %s", err)
+			}
+		}
+	}
+
 	for _, ext := range exts {
 		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
 			file.tempFilename+ext)
@@ -106,6 +121,16 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
 				filepath.Join(file.parent.basePath, file.relativePath+ext)
 		}
+
+		if file.acquireByHash {
+			sums := file.parent.generatedFiles[file.relativePath+ext]
+			for hash, sum := range map[string]string{"SHA512": sums.SHA512, "SHA256": sums.SHA256, "SHA1": sums.SHA1, "MD5Sum": sums.MD5} {
+				err = packageIndexByHash(file, ext, hash, sum)
+				if err != nil {
+					return fmt.Errorf("unable to build hash file: %s", err)
+				}
+			}
+		}
 	}
 
 	if file.signable && signer != nil {
@@ -142,7 +167,53 @@ func (file *indexFile) Finalize(signer utils.Signer) error {
 	return nil
 }
 
-func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+func packageIndexByHash(file *indexFile, ext string, hash string, sum string) error {
+	src := filepath.Join(file.parent.basePath, file.relativePath)
+	indexfile := path.Base(src + ext)
+	src = src + file.parent.suffix + ext
+	filedir := filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath))
+	dst := filepath.Join(filedir, "by-hash", hash)
+	sumfilePath := filepath.Join(dst, sum)
+
+	// link already exists? do nothing
+	exists, err := file.parent.publishedStorage.FileExists(sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error checking exists of file %s: %s", sumfilePath, err)
+	}
+	if exists {
+		return nil
+	}
+
+	// create the link
+	err = file.parent.publishedStorage.HardLink(src, sumfilePath)
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating hardlink %s: %s", sumfilePath, err)
+	}
+
+	// if a previous index file already exists exists, backup symlink
+	if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile)); exists {
+		// if exists, remove old symlink
+		if exists, _ = file.parent.publishedStorage.FileExists(filepath.Join(dst, indexfile+".old")); exists {
+			var link string
+			link, err = file.parent.publishedStorage.ReadLink(filepath.Join(dst, indexfile+".old"))
+			if err != nil {
+				file.parent.publishedStorage.Remove(link)
+			}
+			file.parent.publishedStorage.Remove(filepath.Join(dst, indexfile+".old"))
+		}
+		file.parent.publishedStorage.RenameFile(filepath.Join(dst, indexfile),
+			filepath.Join(dst, indexfile+".old"))
+	}
+
+	// create symlink
+	err = file.parent.publishedStorage.SymLink(filepath.Join(dst, sum), filepath.Join(dst, indexfile))
+	if err != nil {
+		return fmt.Errorf("Acquire-By-Hash: error creating symlink %s: %s", filepath.Join(dst, indexfile), err)
+	}
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
 	return &indexFiles{
 		publishedStorage: publishedStorage,
 		basePath:         basePath,
@@ -151,11 +222,12 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 		tempDir:          tempDir,
 		suffix:           suffix,
 		indexes:          make(map[string]*indexFile),
+		acquireByHash:    acquireByHash,
 	}
 }
 
 func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
@@ -163,7 +235,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Sources")
 		} else {
 			if udeb {
@@ -174,11 +246,12 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  false,
+			discardable:   false,
-			compressable: true,
+			compressable:  true,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -188,7 +261,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
@@ -196,7 +269,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Release")
 		} else {
 			if udeb {
@@ -207,11 +280,12 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  udeb,
+			discardable:   udeb,
-			compressable: false,
+			compressable:  false,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file
@@ -221,7 +295,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)
@@ -236,12 +310,13 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 		}
 
 		file = &indexFile{
-			parent:       files,
+			parent:        files,
-			discardable:  true,
+			discardable:   true,
-			compressable: true,
+			compressable:  true,
-			onlyGzip:     true,
+			onlyGzip:      true,
-			signable:     false,
+			signable:      false,
-			relativePath: relativePath,
+			acquireByHash: files.acquireByHash,
+			relativePath:  relativePath,
 		}
 
 		files.indexes[key] = file

deb/list.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -20,6 +21,8 @@ const (
 	DepFollowAllVariants
 	// DepFollowBuild pulls build dependencies
 	DepFollowBuild
+	// DepVerboseResolve emits additional logs while dependencies are being resolved
+	DepVerboseResolve
 )
 
 // PackageList is list of unique (by key) packages
@@ -31,8 +34,6 @@ const (
 type PackageList struct {
 	// Straight list of packages as map
 	packages map[string]*Package
-	// Has index been prepared?
-	indexed bool
 	// Indexed list of packages, sorted by name internally
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
@@ -41,6 +42,8 @@ type PackageList struct {
 	keyFunc func(p *Package) string
 	// Allow duplicates?
 	duplicatesAllowed bool
+	// Has index been prepared?
+	indexed bool
 }
 
 // PackageConflictError means that package can't be added to the list due to error
@@ -121,6 +124,14 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 	return result, nil
 }
 
+// Has checks whether package is already in the list
+func (l *PackageList) Has(p *Package) bool {
+	key := l.keyFunc(p)
+	_, ok := l.packages[key]
+
+	return ok
+}
+
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
 	key := l.keyFunc(p)
@@ -235,7 +246,7 @@ func (l *PackageList) Remove(p *Package) {
 func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	result = make([]string, 0, 10)
 	for _, pkg := range l.packages {
-		if pkg.Architecture != "all" && (pkg.Architecture != "source" || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
+		if pkg.Architecture != ArchitectureAll && (pkg.Architecture != ArchitectureSource || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
 			result = append(result, pkg.Architecture)
 		}
 	}
@@ -346,6 +357,14 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 		progress.ShutdownBar()
 	}
 
+	if options&DepVerboseResolve == DepVerboseResolve && progress != nil {
+		missingStr := make([]string, len(missing))
+		for i := range missing {
+			missingStr[i] = missing[i].String()
+		}
+		progress.ColoredPrintf("@{y}Missing dependencies:@| %s", strings.Join(missingStr, ", "))
+	}
+
 	return missing, nil
 }
 
@@ -430,18 +449,6 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		panic("list not indexed, can't search")
 	}
 
-	if dep.Relation == VersionDontCare {
-		for _, p := range l.providesIndex[dep.Pkg] {
-			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
-				searchResults = append(searchResults, p)
-
-				if !allMatches {
-					break
-				}
-			}
-		}
-	}
-
 	i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
 
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
@@ -457,11 +464,28 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 		i++
 	}
 
+	if dep.Relation == VersionDontCare {
+		for _, p := range l.providesIndex[dep.Pkg] {
+			if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
+				searchResults = append(searchResults, p)
+
+				if !allMatches {
+					break
+				}
+			}
+		}
+	}
+
 	return
 }
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
 func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+	return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
+}
+
+// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
+func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -488,22 +512,37 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 			added = 0
 
 			// find missing dependencies
-			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, nil)
+			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
 			if err != nil {
 				return nil, err
 			}
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
-				// dependency might have already been satisfied
+				if dependencyOptions&DepFollowAllVariants == 0 {
-				// with packages already been added
+					// dependency might have already been satisfied
-				if result.Search(dep, false) != nil {
+					// with packages already been added
-					continue
+					//
+					// when follow-all-variants is enabled, we need to try to expand anyway,
+					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					if result.Search(dep, false) != nil {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true))
+						}
+						continue
+					}
 				}
 
-				searchResults := l.Search(dep, false)
+				searchResults := l.Search(dep, true)
-				if searchResults != nil {
+				if len(searchResults) > 0 {
 					for _, p := range searchResults {
+						if result.Has(p) {
+							continue
+						}
+
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
+						}
 						result.Add(p)
 						dependencySource.Add(p)
 						added++
@@ -511,6 +550,11 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 							break
 						}
 					}
+				} else {
+					if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+						progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
+					}
+
 				}
 			}
 		}

deb/local_test.go

@@ -18,7 +18,7 @@ type LocalRepoSuite struct {
 var _ = Suite(&LocalRepoSuite{})
 
 func (s *LocalRepoSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.list = NewPackageList()
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +83,7 @@ type LocalRepoCollectionSuite struct {
 var _ = Suite(&LocalRepoCollectionSuite{})
 
 func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewLocalRepoCollection(s.db)
 
 	s.list = NewPackageList()
@@ -98,14 +98,14 @@ func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("local1")
+	_, err := s.collection.ByName("local1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("local1")
+	r, err := s.collection.ByName("local1")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -116,13 +116,13 @@ func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/package.go

@@ -24,12 +24,12 @@ type Package struct {
 	Source string
 	// List of virtual packages this package provides
 	Provides []string
+	// Hash of files section
+	FilesHash uint64
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
 	IsUdeb bool
-	// Hash of files section
-	FilesHash uint64
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
@@ -41,6 +41,20 @@ type Package struct {
 	collection *PackageCollection
 }
 
+// Package types
+const (
+	PackageTypeBinary = "deb"
+	PackageTypeUdeb   = "udeb"
+	PackageTypeSource = "source"
+)
+
+// Special arhictectures
+const (
+	ArchitectureAll    = "all"
+	ArhictectureAny    = "any"
+	ArchitectureSource = "source"
+)
+
 // Check interface
 var (
 	_ json.Marshaler = &Package{}
@@ -218,12 +232,12 @@ func (p *Package) GetField(name string) string {
 		return p.Architecture
 	case "$PackageType":
 		if p.IsSource {
-			return "source"
+			return PackageTypeSource
 		}
 		if p.IsUdeb {
-			return "udeb"
+			return PackageTypeUdeb
 		}
-		return "deb"
+		return PackageTypeBinary
 	case "Name":
 		return p.Name
 	case "Version":
@@ -256,7 +270,7 @@ func (p *Package) GetField(name string) string {
 
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
-	if p.Architecture == "all" && arch != "source" {
+	if p.Architecture == ArchitectureAll && arch != ArchitectureSource {
 		return true
 	}
 
@@ -344,7 +358,7 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 		if source == "" {
 			source = p.Name
 		}
-		if strings.Index(source, ")") != -1 {
+		if strings.Contains(source, ")") {
 			dependencies = append(dependencies, fmt.Sprintf("%s {source}", source))
 		} else {
 			dependencies = append(dependencies, fmt.Sprintf("%s (= %s) {source}", source, p.Version))
@@ -403,32 +417,47 @@ func (p *Package) Files() PackageFiles {
 }
 
 // Contents returns cached package contents
-func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+func (p *Package) Contents(packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	if p.IsSource {
 		return nil
 	}
 
-	return p.collection.loadContents(p, packagePool)
+	return p.collection.loadContents(p, packagePool, progress)
 }
 
 // CalculateContents looks up contents in package file
-func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptly.Progress) ([]string, error) {
 	if p.IsSource {
-		return nil
+		return nil, nil
 	}
 
 	file := p.Files()[0]
-	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	poolPath, err := file.GetPoolPath(packagePool)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to build pool path: @| %s", err)
+		}
+		return nil, err
 	}
 
-	contents, err := GetContentsFromDeb(path)
+	reader, err := packagePool.Open(poolPath)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to open package in pool: @| %s", err)
+		}
+		return nil, err
+	}
+	defer reader.Close()
+
+	contents, err := GetContentsFromDeb(reader, file.Filename)
+	if err != nil {
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to generate package contents: @| %s", err)
+		}
+		return nil, err
 	}
 
-	return contents
+	return contents, nil
 }
 
 // UpdateFiles saves new state of files
@@ -541,7 +570,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 	}
 
 	for i, f := range p.Files() {
-		sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+		sourcePoolPath, err := f.GetPoolPath(packagePool)
 		if err != nil {
 			return err
 		}
@@ -549,7 +578,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
-		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
+		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
 		if err != nil {
 			return err
 		}
@@ -590,29 +619,26 @@ func (p *Package) PoolDirectory() (string, error) {
 
 // PackageDownloadTask is a element of download queue for the package
 type PackageDownloadTask struct {
-	RepoURI         string
+	File         *PackageFile
-	DestinationPath string
+	Additional   []PackageDownloadTask
-	Checksums       utils.ChecksumInfo
+	TempDownPath string
+	Done         bool
 }
 
 // DownloadList returns list of missing package files for download in format
 // [[srcpath, dstpath]]
-func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
+func (p *Package) DownloadList(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result []PackageDownloadTask, err error) {
 	result = make([]PackageDownloadTask, 0, 1)
 
-	for _, f := range p.Files() {
+	files := p.Files()
-		poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+	for idx := range files {
-		if err != nil {
+		verified, err := files[idx].Verify(packagePool, checksumStorage)
-			return nil, err
-		}
-
-		verified, err := f.Verify(packagePool)
 		if err != nil {
 			return nil, err
 		}
 
 		if !verified {
-			result = append(result, PackageDownloadTask{RepoURI: f.DownloadURL(), DestinationPath: poolPath, Checksums: f.Checksums})
+			result = append(result, PackageDownloadTask{File: &files[idx]})
 		}
 	}
 
@@ -620,11 +646,11 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD
 }
 
 // VerifyFiles verifies that all package files have neen correctly downloaded
-func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
+func (p *Package) VerifyFiles(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result bool, err error) {
 	result = true
 
 	for _, f := range p.Files() {
-		result, err = f.Verify(packagePool)
+		result, err = f.Verify(packagePool, checksumStorage)
 		if err != nil || !result {
 			return
 		}
@@ -639,7 +665,7 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error)
 	result := make([]string, len(p.Files()))
 
 	for i, f := range p.Files() {
-		result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
+		result[i], err = f.GetPoolPath(packagePool)
 		if err != nil {
 			return nil, err
 		}

deb/package_collection.go

@@ -163,7 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 }
 
 // loadContents loads or calculates and saves package contents
-func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	encoded, err := collection.db.Get(p.Key("xC"))
 	if err == nil {
 		contents := []string{}
@@ -181,7 +181,11 @@ func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.
 		panic("unable to load contents")
 	}
 
-	contents := p.CalculateContents(packagePool)
+	contents, err := p.CalculateContents(packagePool, progress)
+	if err != nil {
+		// failed to acquire contents, don't persist it
+		return contents
+	}
 
 	var buf bytes.Buffer
 	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)
@@ -311,7 +315,7 @@ func (collection *PackageCollection) SearchSupported() bool {
 
 // SearchByKey finds package by exact key
 func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
-	result = NewPackageList()
+	result = NewPackageListWithDuplicates(true, 0)
 
 	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
 		pkg, err := collection.ByKey(key)

deb/package_collection_test.go

@@ -17,7 +17,7 @@ var _ = Suite(&PackageCollectionSuite{})
 
 func (s *PackageCollectionSuite) SetUpTest(c *C) {
 	s.p = NewPackageFromControlFile(packageStanza.Copy())
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewPackageCollection(s.db)
 }
 

deb/package_files.go

@@ -4,7 +4,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"hash/fnv"
-	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -20,25 +19,33 @@ type PackageFile struct {
 	Filename string
 	// Hashes for the file
 	Checksums utils.ChecksumInfo
+	// PoolPath persists relative path to file in the package pool
+	PoolPath string
 	// Temporary field used while downloading, stored relative path on the mirror
 	downloadPath string
 }
 
 // Verify that package file is present and correct
-func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
+func (f *PackageFile) Verify(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (bool, error) {
-	poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+	generatedPoolPath, exists, err := packagePool.Verify(f.PoolPath, f.Filename, &f.Checksums, checksumStorage)
-	if err != nil {
+	if exists && err == nil {
-		return false, err
+		f.PoolPath = generatedPoolPath
 	}
 
-	st, err := os.Stat(poolPath)
+	return exists, err
-	if err != nil {
+}
-		return false, nil
+
+// GetPoolPath returns path to the file in the pool
+//
+// For legacy packages which do not have PoolPath field set, that calculates LegacyPath via pool
+func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) {
+	var err error
+
+	if f.PoolPath == "" {
+		f.PoolPath, err = packagePool.LegacyPath(f.Filename, &f.Checksums)
 	}
 
-	// verify size
+	return f.PoolPath, err
-	// TODO: verify checksum if configured
-	return st.Size() == f.Checksums.Size, nil
 }
 
 // DownloadURL return relative URL to package download location

deb/package_files_test.go

@@ -1,9 +1,10 @@
 package deb
 
 import (
-	"os"
+	"io/ioutil"
 	"path/filepath"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
 
@@ -12,11 +13,13 @@ import (
 
 type PackageFilesSuite struct {
 	files PackageFiles
+	cs    aptly.ChecksumStorage
 }
 
 var _ = Suite(&PackageFilesSuite{})
 
 func (s *PackageFilesSuite) SetUpTest(c *C) {
+	s.cs = files.NewMockChecksumStorage()
 	s.files = PackageFiles{PackageFile{
 		Filename:     "alien-arena-common_7.40-2_i386.deb",
 		downloadPath: "pool/contrib/a/alien-arena",
@@ -29,27 +32,24 @@ func (s *PackageFilesSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageFilesSuite) TestVerify(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums.MD5)
 
-	result, err := s.files[0].Verify(packagePool)
+	result, err := s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
+	s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
 
-	result, err = s.files[0].Verify(packagePool)
+	s.files[0].Checksums.Size = 187518
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	s.files[0].Checksums.Size = 5
-	result, err = s.files[0].Verify(packagePool)
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/package_test.go

@@ -2,12 +2,11 @@ package deb
 
 import (
 	"bytes"
-	"os"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 
 	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
@@ -300,7 +299,7 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// ~
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
-			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+			Regexp: regexp.MustCompile(`7\.40-.*`)}), Equals, true)
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
 			Regexp: regexp.MustCompile("40")}), Equals, true)
@@ -363,19 +362,17 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 }
 
 func (s *PackageSuite) TestLinkFromPool(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	publishedStorage := files.NewPublishedStorage(c.MkDir())
+	cs := files.NewMockChecksumStorage()
+	publishedStorage := files.NewPublishedStorage(c.MkDir(), "", "")
 	p := NewPackageFromControlFile(s.stanza)
 
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
-	c.Assert(err, IsNil)
 
-	file, err := os.Create(poolPath)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
-	c.Assert(err, IsNil)
-	file.Close()
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
@@ -387,7 +384,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 }
 
 func (s *PackageSuite) TestFilepathList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), true)
 	p := NewPackageFromControlFile(s.stanza)
 
 	list, err := p.FilepathList(packagePool)
@@ -396,31 +393,24 @@ func (s *PackageSuite) TestFilepathList(c *C) {
 }
 
 func (s *PackageSuite) TestDownloadList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
+	cs := files.NewMockChecksumStorage()
 	p := NewPackageFromControlFile(s.stanza)
 	p.Files()[0].Checksums.Size = 5
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 
-	list, err := p.DownloadList(packagePool)
+	list, err := p.DownloadList(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
 		{
-			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
+			File: &p.Files()[0],
-			DestinationPath: poolPath,
+		},
-			Checksums: utils.ChecksumInfo{Size: 5,
+	})
-				MD5:    "1e8cba92c41420aa7baa8a5718d67122",
-				SHA1:   "46955e48cad27410a83740a21d766ce362364024",
-				SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	file, err := os.Create(poolPath)
+	list, err = p.DownloadList(packagePool, cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
-
-	list, err = p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{})
 }
@@ -428,24 +418,22 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 func (s *PackageSuite) TestVerifyFiles(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
+	cs := files.NewMockChecksumStorage()
 
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
 
-	result, err := p.VerifyFiles(packagePool)
+	p.Files()[0].Checksums.Size = 100
+	result, err := p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	p.Files()[0].Checksums.Size = 5
 
-	result, err = p.VerifyFiles(packagePool)
+	result, err = p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/publish.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -36,17 +37,17 @@ type PublishedRepo struct {
 	// Internal unique ID
 	UUID string
 	// Storage & Prefix & distribution should be unique across all published repositories
-	Storage      string
+	Storage              string
-	Prefix       string
+	Prefix               string
-	Distribution string
+	Distribution         string
-	Origin       string
+	Origin               string
-	Label        string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
+	Label                string
 	// Architectures is a list of all architectures published
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
-	// Skip contents generation
-	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -55,12 +56,17 @@ type PublishedRepo struct {
 	Component string
 	// SourceUUID is UUID of either snapshot or local repo
 	SourceUUID string `codec:"SnapshotUUID"`
-
 	// Map of component to source items
 	sourceItems map[string]repoSourceItem
 
+	// Skip contents generation
+	SkipContents bool
+
 	// True if repo is being re-published
 	rePublishing bool
+
+	// Provide index files per hash also
+	AcquireByHash bool
 }
 
 // ParsePrefix splits [storage:]prefix into components
@@ -75,6 +81,7 @@ func ParsePrefix(param string) (storage, prefix string) {
 	} else {
 		prefix = param
 	}
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
 	return
 }
 
@@ -96,19 +103,19 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 
 		if snapshot, ok := head.(*Snapshot); ok {
 			for _, uuid := range snapshot.SourceIDs {
-				if snapshot.SourceKind == "repo" {
+				if snapshot.SourceKind == SourceRemoteRepo {
 					remoteRepo, err := collectionFactory.RemoteRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, remoteRepo)
-				} else if snapshot.SourceKind == "local" {
+				} else if snapshot.SourceKind == SourceLocalRepo {
 					localRepo, err := collectionFactory.LocalRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, localRepo)
-				} else if snapshot.SourceKind == "snapshot" {
+				} else if snapshot.SourceKind == SourceSnapshot {
 					snap, err := collectionFactory.SnapshotCollection().ByUUID(uuid)
 					if err != nil {
 						continue
@@ -166,23 +173,20 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		component               string
 		snapshot                *Snapshot
 		localRepo               *LocalRepo
-		ok                      bool
+		fields                  = make(map[string][]string)
 	)
 
 	// get first source
 	source = sources[0]
 
 	// figure out source kind
-	snapshot, ok = source.(*Snapshot)
+	switch source.(type) {
-	if ok {
+	case *Snapshot:
-		result.SourceKind = "snapshot"
+		result.SourceKind = SourceSnapshot
-	} else {
+	case *LocalRepo:
-		localRepo, ok = source.(*LocalRepo)
+		result.SourceKind = SourceLocalRepo
-		if ok {
+	default:
-			result.SourceKind = "local"
+		panic("unknown source kind")
-		} else {
-			panic("unknown source kind")
-		}
 	}
 
 	for i := range sources {
@@ -213,11 +217,21 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 			return nil, fmt.Errorf("duplicate component name: %s", component)
 		}
 
-		if result.SourceKind == "snapshot" {
+		if result.SourceKind == SourceSnapshot {
 			snapshot = source.(*Snapshot)
 			result.Sources[component] = snapshot.UUID
 			result.sourceItems[component] = repoSourceItem{snapshot: snapshot}
-		} else if result.SourceKind == "local" {
+
+			if !utils.StrSliceHasItem(fields["Origin"], snapshot.Origin) {
+				fields["Origin"] = append(fields["Origin"], snapshot.Origin)
+			}
+			if !utils.StrSliceHasItem(fields["NotAutomatic"], snapshot.NotAutomatic) {
+				fields["NotAutomatic"] = append(fields["NotAutomatic"], snapshot.NotAutomatic)
+			}
+			if !utils.StrSliceHasItem(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades) {
+				fields["ButAutomaticUpgrades"] = append(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades)
+			}
+		} else if result.SourceKind == SourceLocalRepo {
 			localRepo = source.(*LocalRepo)
 			result.Sources[component] = localRepo.UUID
 			result.sourceItems[component] = repoSourceItem{localRepo: localRepo, packageRefs: localRepo.RefList()}
@@ -226,12 +240,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 
 	// clean & verify prefix
 	prefix = filepath.Clean(prefix)
-	if strings.HasPrefix(prefix, "/") {
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
-		prefix = prefix[1:]
-	}
-	if strings.HasSuffix(prefix, "/") {
-		prefix = prefix[:len(prefix)-1]
-	}
 	prefix = filepath.Clean(prefix)
 
 	for _, part := range strings.Split(prefix, "/") {
@@ -252,12 +261,23 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		}
 	}
 
-	if strings.Index(distribution, "/") != -1 {
+	if strings.Contains(distribution, "/") {
 		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
 	}
 
 	result.Distribution = distribution
 
+	// only fields which are unique by all given snapshots are set on published
+	if len(fields["Origin"]) == 1 {
+		result.Origin = fields["Origin"][0]
+	}
+	if len(fields["NotAutomatic"]) == 1 {
+		result.NotAutomatic = fields["NotAutomatic"][0]
+	}
+	if len(fields["ButAutomaticUpgrades"]) == 1 {
+		result.ButAutomaticUpgrades = fields["ButAutomaticUpgrades"][0]
+	}
+
 	return result, nil
 }
 
@@ -284,15 +304,17 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 	}
 
 	return json.Marshal(map[string]interface{}{
-		"Architectures": p.Architectures,
+		"Architectures":        p.Architectures,
-		"Distribution":  p.Distribution,
+		"Distribution":         p.Distribution,
-		"Label":         p.Label,
+		"Label":                p.Label,
-		"Origin":        p.Origin,
+		"Origin":               p.Origin,
-		"Prefix":        p.Prefix,
+		"NotAutomatic":         p.NotAutomatic,
-		"SourceKind":    p.SourceKind,
+		"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
-		"Sources":       sources,
+		"Prefix":               p.Prefix,
-		"Storage":       p.Storage,
+		"SourceKind":           p.SourceKind,
-		"SkipContents":  p.SkipContents,
+		"Sources":              sources,
+		"Storage":              p.Storage,
+		"SkipContents":         p.SkipContents,
 	})
 }
 
@@ -315,19 +337,27 @@ func (p *PublishedRepo) String() string {
 		sources = append(sources, fmt.Sprintf("{%s: %s}", component, source))
 	}
 
+	var extras []string
 	var extra string
 
 	if p.Origin != "" {
-		extra += fmt.Sprintf("origin: %s", p.Origin)
+		extras = append(extras, fmt.Sprintf("origin: %s", p.Origin))
+	}
+
+	if p.NotAutomatic != "" {
+		extras = append(extras, fmt.Sprintf("notautomatic: %s", p.NotAutomatic))
+	}
+
+	if p.ButAutomaticUpgrades != "" {
+		extras = append(extras, fmt.Sprintf("butautomaticupgrades: %s", p.ButAutomaticUpgrades))
 	}
 
 	if p.Label != "" {
-		if extra != "" {
+		extras = append(extras, fmt.Sprintf("label: %s", p.Label))
-			extra += ", "
-		}
-		extra += fmt.Sprintf("label: %s", p.Label)
 	}
 
+	extra = strings.Join(extras, ", ")
+
 	if extra != "" {
 		extra = " (" + extra + ")"
 	}
@@ -358,10 +388,10 @@ func (p *PublishedRepo) RefKey(component string) []byte {
 // RefList returns list of package refs in local repo
 func (p *PublishedRepo) RefList(component string) *PackageRefList {
 	item := p.sourceItems[component]
-	if p.SourceKind == "local" {
+	if p.SourceKind == SourceLocalRepo {
 		return item.packageRefs
 	}
-	if p.SourceKind == "snapshot" {
+	if p.SourceKind == SourceSnapshot {
 		return item.snapshot.RefList()
 	}
 	panic("unknown source")
@@ -380,7 +410,7 @@ func (p *PublishedRepo) Components() []string {
 
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
-	if p.SourceKind != "local" {
+	if p.SourceKind != SourceLocalRepo {
 		panic("not local repo publish")
 	}
 
@@ -393,7 +423,7 @@ func (p *PublishedRepo) UpdateLocalRepo(component string) {
 
 // UpdateSnapshot switches snapshot for component
 func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) {
-	if p.SourceKind != "snapshot" {
+	if p.SourceKind != SourceSnapshot {
 		panic("not snapshot publish")
 	}
 
@@ -425,7 +455,7 @@ func (p *PublishedRepo) Decode(input []byte) error {
 
 	// old PublishedRepo were publishing only snapshots
 	if p.SourceKind == "" {
-		p.SourceKind = "snapshot"
+		p.SourceKind = SourceSnapshot
 	}
 
 	// <0.6 aptly used single SourceUUID + Component instead of Sources
@@ -456,7 +486,7 @@ func (p *PublishedRepo) GetLabel() string {
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
-	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
 
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
@@ -473,8 +503,16 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	if err != nil {
 		return err
 	}
-	defer tempDB.Close()
+	defer func() {
-	defer tempDB.Drop()
+		e := tempDB.Close()
+		if e != nil && progress != nil {
+			progress.Printf("failed to close temp DB: %s", err)
+		}
+		e = tempDB.Drop()
+		if e != nil && progress != nil {
+			progress.Printf("failed to drop temp DB: %s", err)
+		}
+	}()
 
 	if progress != nil {
 		progress.Printf("Loading packages...\n")
@@ -521,7 +559,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
-	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
 
 	for component, list := range lists {
 		hadUdebs := false
@@ -574,7 +612,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 							contentIndexes[key] = contentIndex
 						}
 
-						contentIndex.Push(pkg, packagePool)
+						contentIndex.Push(pkg, packagePool, progress)
 					}
 
 					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
@@ -612,7 +650,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					continue
 				}
 
-				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				var bufWriter *bufio.Writer
+				bufWriter, err = indexes.ContentsIndex(component, arch, udeb).BufWriter()
 				if err != nil {
 					return fmt.Errorf("unable to generate contents index: %v", err)
 				}
@@ -647,6 +686,9 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				release["Component"] = component
 				release["Origin"] = p.GetOrigin()
 				release["Label"] = p.GetLabel()
+				if p.AcquireByHash {
+					release["Acquire-By-Hash"] = "yes"
+				}
 
 				var bufWriter *bufio.Writer
 				bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -673,11 +715,20 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
+	if p.NotAutomatic != "" {
+		release["NotAutomatic"] = p.NotAutomatic
+	}
+	if p.ButAutomaticUpgrades != "" {
+		release["ButAutomaticUpgrades"] = p.ButAutomaticUpgrades
+	}
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.Distribution
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
-	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
+	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
+	if p.AcquireByHash {
+		release["Acquire-By-Hash"] = "yes"
+	}
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""
@@ -721,12 +772,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = indexes.RenameFiles()
+	return indexes.RenameFiles()
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
 // RemoveFiles removes files that were created by Publish
@@ -825,7 +871,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 		return
 	}
 
-	if repo.SourceKind == "local" {
+	if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
 			err = collection.db.Put(repo.RefKey(component), item.packageRefs.Encode())
 			if err != nil {
@@ -840,7 +886,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
 	repo.sourceItems = make(map[string]repoSourceItem)
 
-	if repo.SourceKind == "snapshot" {
+	if repo.SourceKind == SourceSnapshot {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -855,7 +901,7 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 			repo.sourceItems[component] = item
 		}
-	} else if repo.SourceKind == "local" {
+	} else if repo.SourceKind == SourceLocalRepo {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -923,7 +969,7 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "snapshot" {
+		if r.SourceKind == SourceSnapshot {
 			if r.SourceUUID == snapshot.UUID {
 				result = append(result, r)
 			}
@@ -943,7 +989,7 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "local" {
+		if r.SourceKind == SourceLocalRepo {
 			if r.SourceUUID == repo.UUID {
 				result = append(result, r)
 			}
@@ -1060,7 +1106,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 // Remove removes published repository, cleaning up directories, files
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
-	force bool) error {
+	force, skipCleanup bool) error {
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1097,7 +1143,7 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
 		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
 
-	if len(cleanComponents) > 0 {
+	if !skipCleanup && len(cleanComponents) > 0 {
 		err = collection.CleanupPrefixComponentFiles(repo.Prefix, cleanComponents,
 			publishedStorageProvider.GetPublishedStorage(storage), collectionFactory, progress)
 		if err != nil {

deb/publish_test.go

@@ -74,6 +74,7 @@ type PublishedRepoSuite struct {
 	provider                            *FakeStorageProvider
 	publishedStorage, publishedStorage2 *files.PublishedStorage
 	packagePool                         aptly.PackagePool
+	cs                                  aptly.ChecksumStorage
 	localRepo                           *LocalRepo
 	snapshot, snapshot2                 *Snapshot
 	db                                  database.Storage
@@ -86,17 +87,31 @@ var _ = Suite(&PublishedRepoSuite{})
 func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
 
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
 		"":            s.publishedStorage,
 		"files:other": s.publishedStorage2}}
-	s.packagePool = files.NewPackagePool(s.root)
+	s.packagePool = files.NewPackagePool(s.root, false)
+	s.cs = files.NewMockChecksumStorage()
+
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+
+	var err error
+	s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs)
+	c.Assert(err, IsNil)
+
+	s.p1.UpdateFiles(s.p1.Files())
+	s.p2.UpdateFiles(s.p1.Files())
+	s.p3.UpdateFiles(s.p1.Files())
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
@@ -131,12 +146,6 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	s.repo5.SkipContents = true
-
-	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	f, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	f.Close()
 }
 
 func (s *PublishedRepoSuite) TearDownTest(c *C) {
@@ -268,7 +277,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
@@ -292,7 +301,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
@@ -440,7 +449,7 @@ type PublishedRepoCollectionSuite struct {
 var _ = Suite(&PublishedRepoCollectionSuite{})
 
 func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -468,7 +477,7 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
-	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	_, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
@@ -480,7 +489,7 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(s.collection.Add(s.repo4), IsNil)
 	c.Assert(s.collection.Add(s.repo5), IsNil)
 
-	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -496,16 +505,17 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(r.String(), Equals, s.repo1.String())
 
 	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Assert(err, IsNil)
 	c.Check(r.String(), Equals, s.repo5.String())
 }
 
 func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID(s.repo1.UUID)
+	_, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
 
-	r, err = s.collection.ByUUID(s.repo1.UUID)
+	r, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -552,7 +562,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 		Prefix:        "ppa",
 		Distribution:  "anaconda",
 		Architectures: []string{"i386"},
-		SourceKind:    "local",
+		SourceKind:    SourceLocalRepo,
 		Component:     "contrib",
 		SourceUUID:    s.localRepo.UUID,
 	}
@@ -630,7 +640,7 @@ type PublishedRepoRemoveSuite struct {
 var _ = Suite(&PublishedRepoRemoveSuite{})
 
 func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -653,7 +663,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.publishedStorage.MkDir("ppa/dists/anaconda")
 	s.publishedStorage.MkDir("ppa/dists/meduza")
 	s.publishedStorage.MkDir("ppa/dists/osminog")
@@ -663,7 +673,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.publishedStorage.MkDir("pool/main")
 
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.publishedStorage2.MkDir("ppa/dists/osminog")
 	s.publishedStorage2.MkDir("ppa/pool/contrib")
 
@@ -746,7 +756,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
@@ -766,10 +776,48 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
 
-	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, false)
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false)
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, false)
+	c.Check(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+}
+
+func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2SkipCleanup(c *C) {
+	err := s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	collection := NewPublishedRepoCollection(s.db)
+	_, err = collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	c.Check(err, ErrorMatches, ".*not found")
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage2.PublicPath(), "ppa/pool/contrib"), PathExists)
+
+	err = s.collection.Remove(s.provider, "", "ppa", "anaconda", s.factory, nil, false, true)
+	c.Check(err, ErrorMatches, ".*not found")
+
+	err = s.collection.Remove(s.provider, "", "ppa", "meduza", s.factory, nil, false, true)
 	c.Check(err, IsNil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
@@ -784,7 +832,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "", ".", "anaconda", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("", ".", "anaconda")
@@ -806,7 +854,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo5(c *C) {
-	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false)
+	err := s.collection.Remove(s.provider, "files:other", "ppa", "osminog", s.factory, nil, false, false)
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "osminog")

deb/query.go

@@ -204,7 +204,7 @@ func (q *FieldQuery) Fast(list PackageCatalog) bool {
 // String interface
 func (q *FieldQuery) String() string {
 	escape := func(val string) string {
-		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+		if strings.ContainsAny(val, "()|,!{} \t\n") {
 			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
 		}
 		return val

deb/reflist.go

@@ -92,7 +92,7 @@ func (l *PackageRefList) Has(p *Package) bool {
 	key := p.Key("")
 
 	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
-	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+	return i < len(l.Refs) && bytes.Equal(l.Refs[i], key)
 }
 
 // Strings builds list of strings with package keys
@@ -395,6 +395,4 @@ func (l *PackageRefList) FilterLatestRefs() {
 
 		lastArch, lastName, lastVer = arch, name, ver
 	}
-
-	return
 }

deb/reflist_test.go

@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 	coll.Update(s.p1)
 	coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
 }
 
 func (s *PackageRefListSuite) TestDiff(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 }
 
 func (s *PackageRefListSuite) TestMerge(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{

deb/remote.go

@@ -2,12 +2,12 @@ package deb
 
 import (
 	"bytes"
+	gocontext "context"
 	"fmt"
 	"log"
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -18,6 +18,7 @@ import (
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
@@ -45,10 +46,6 @@ type RemoteRepo struct {
 	Components []string
 	// List of architectures to fetch, if empty, then fetch all architectures
 	Architectures []string
-	// Should we download sources?
-	DownloadSources bool
-	// Should we download .udebs?
-	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
@@ -57,20 +54,22 @@ type RemoteRepo struct {
 	ReleaseFiles map[string]utils.ChecksumInfo
 	// Filter for packages
 	Filter string
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
 	// SkipComponentCheck skips component list verification
 	SkipComponentCheck bool
 	// SkipArchitectureCheck skips architecture list verification
 	SkipArchitectureCheck bool
-	// Status marks state of repository (being updated, no action)
+	// Should we download sources?
-	Status int
+	DownloadSources bool
-	// WorkerPID is PID of the process modifying the mirror (if any)
+	// Should we download .udebs?
-	WorkerPID int
+	DownloadUdebs bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
-	// Temporary list of package refs
-	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
 	// Current list of packages (filled while updating mirror)
@@ -114,6 +113,12 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 	return result, nil
 }
 
+// SetArchiveRoot of remote repo
+func (repo *RemoteRepo) SetArchiveRoot(archiveRoot string) {
+	repo.ArchiveRoot = archiveRoot
+	repo.prepare()
+}
+
 func (repo *RemoteRepo) prepare() error {
 	var err error
 
@@ -193,49 +198,49 @@ func (repo *RemoteRepo) CheckLock() error {
 	return nil
 }
 
-// ReleaseURL returns URL to Release* files in repo root
+// IndexesRootURL builds URL for various indexes
-func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
+func (repo *RemoteRepo) IndexesRootURL() *url.URL {
 	var path *url.URL
 
 	if !repo.IsFlat() {
-		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
+		path = &url.URL{Path: fmt.Sprintf("dists/%s/", repo.Distribution)}
 	} else {
-		path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
+		path = &url.URL{Path: repo.Distribution}
 	}
 
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// FlatBinaryURL returns URL to Packages files for flat repo
+// ReleaseURL returns URL to Release* files in repo root
-func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
+func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
+	return repo.IndexesRootURL().ResolveReference(&url.URL{Path: name})
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// FlatSourcesURL returns URL to Sources files for flat repo
+// FlatBinaryPath returns path to Packages files for flat repo
-func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
+func (repo *RemoteRepo) FlatBinaryPath() string {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
+	return "Packages"
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// BinaryURL returns URL of Packages files for given component and
+// FlatSourcesPath returns path to Sources files for flat repo
+func (repo *RemoteRepo) FlatSourcesPath() string {
+	return "Sources"
+}
+
+// BinaryPath returns path to Packages files for given component and
 // architecture
-func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
+func (repo *RemoteRepo) BinaryPath(component string, architecture string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return fmt.Sprintf("%s/binary-%s/Packages", component, architecture)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// SourcesURL returns URL of Sources files for given component
+// SourcesPath returns path to Sources files for given component
-func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
+func (repo *RemoteRepo) SourcesPath(component string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/source/Sources", repo.Distribution, component)}
+	return fmt.Sprintf("%s/source/Sources", component)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// UdebURL returns URL of Packages files for given component and
+// UdebPath returns path of Packages files for given component and
 // architecture
-func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
+func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
 // PackageURL returns URL of package file relative to repository root
@@ -246,7 +251,7 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 }
 
 // Fetch updates information about repository
-func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
+func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 	var (
 		release, inrelease, releasesig *os.File
 		err                            error
@@ -254,13 +259,13 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 
 	if verifier == nil {
 		// 0. Just download release file to temporary URL
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 	} else {
 		// 1. try InRelease file
-		inrelease, err = http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		inrelease, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("InRelease").String())
 		if err != nil {
 			goto splitsignature
 		}
@@ -282,17 +287,17 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 
 	splitsignature:
 		// 2. try Release + Release.gpg
-		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
 		if err != nil {
 			return err
 		}
 
-		releasesig, err = http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		releasesig, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release.gpg").String())
 		if err != nil {
 			return err
 		}
 
-		err = verifier.VerifyDetachedSignature(releasesig, release)
+		err = verifier.VerifyDetachedSignature(releasesig, release, true)
 		if err != nil {
 			return err
 		}
@@ -316,7 +321,7 @@ ok:
 		architectures := strings.Split(stanza["Architectures"], " ")
 		sort.Strings(architectures)
 		// "source" architecture is never present, despite Release file claims
-		architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
+		architectures = utils.StrSlicesSubstract(architectures, []string{ArchitectureSource})
 		if len(repo.Architectures) == 0 {
 			repo.Architectures = architectures
 		} else if !repo.SkipArchitectureCheck {
@@ -331,9 +336,7 @@ ok:
 		if strings.Contains(repo.Distribution, "/") {
 			distributionLast := path.Base(repo.Distribution) + "/"
 			for i := range components {
-				if strings.HasPrefix(components[i], distributionLast) {
+				components[i] = strings.TrimPrefix(components[i], distributionLast)
-					components[i] = components[i][len(distributionLast):]
-				}
 			}
 		}
 		if len(repo.Components) == 0 {
@@ -395,7 +398,10 @@ ok:
 		return err
 	}
 
-	delete(stanza, "SHA512")
+	err = parseSums("SHA512", func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	if err != nil {
+		return err
+	}
 
 	repo.Meta = stanza
 
@@ -411,30 +417,30 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
-	packagesURLs := [][]string{}
+	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
 		if repo.DownloadSources {
-			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
 				if repo.DownloadUdebs {
-					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
 				}
 			}
 			if repo.DownloadSources {
-				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
 			}
 		}
 	}
 
-	for _, info := range packagesURLs {
+	for _, info := range packagesPaths {
-		url, kind := info[0], info[1]
+		path, kind := info[0], info[1]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
 		if err != nil {
 			return err
 		}
@@ -459,11 +465,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 			var p *Package
 
-			if kind == "binary" {
+			if kind == PackageTypeBinary {
 				p = NewPackageFromControlFile(stanza)
-			} else if kind == "udeb" {
+			} else if kind == PackageTypeUdeb {
 				p = NewUdebPackageFromControlFile(stanza)
-			} else if kind == "source" {
+			} else if kind == PackageTypeSource {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
@@ -477,11 +483,6 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					return err
 				}
 			}
-
-			err = collectionFactory.PackageCollection().Update(p)
-			if err != nil {
-				return err
-			}
 		}
 
 		progress.ShutdownBar()
@@ -491,14 +492,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 }
 
 // ApplyFilter applies filtering to already built PackageList
-func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery, progress aptly.Progress) (oldLen, newLen int, err error) {
 	repo.packageList.PrepareIndex()
 
 	emptyList := NewPackageList()
 	emptyList.PrepareIndex()
 
 	oldLen = repo.packageList.Len()
-	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress)
 	if repo.packageList != nil {
 		newLen = repo.packageList.Len()
 	}
@@ -506,24 +507,40 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 }
 
 // BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
-	seen := make(map[string]struct{}, repo.packageList.Len())
+	seen := make(map[string]int, repo.packageList.Len())
 
 	err = repo.packageList.ForEach(func(p *Package) error {
-		list, err2 := p.DownloadList(packagePool)
+		if repo.packageRefs != nil && skipExistingPackages {
+			if repo.packageRefs.Has(p) {
+				// skip this package, but load checksums/files from package in DB
+				var prevP *Package
+				prevP, err = packageCollection.ByKey(p.Key(""))
+				if err != nil {
+					return err
+				}
+
+				p.UpdateFiles(prevP.Files())
+				return nil
+			}
+		}
+
+		list, err2 := p.DownloadList(packagePool, checksumStorage)
 		if err2 != nil {
 			return err2
 		}
-		p.files = nil
 
 		for _, task := range list {
-			key := task.RepoURI + "-" + task.DestinationPath
+			key := task.File.DownloadURL()
-			_, found := seen[key]
+			idx, found := seen[key]
 			if !found {
 				queue = append(queue, task)
-				downloadSize += task.Checksums.Size
+				downloadSize += task.File.Checksums.Size
-				seen[key] = struct{}{}
+				seen[key] = len(queue) - 1
+			} else {
+				// hook up the task to duplicate entry already on the list
+				queue[idx].Additional = append(queue[idx].Additional, task)
 			}
 		}
 
@@ -533,17 +550,39 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue
 		return
 	}
 
-	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
-	// free up package list, we don't need it after this point
-	repo.packageList = nil
-
 	return
 }
 
 // FinalizeDownload swaps for final value of package refs
-func (repo *RemoteRepo) FinalizeDownload() {
+func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
 	repo.LastDownloadDate = time.Now()
-	repo.packageRefs = repo.tempPackageRefs
+
+	if progress != nil {
+		progress.InitBar(int64(repo.packageList.Len()), true)
+	}
+
+	var i int
+
+	// update all the packages in collection
+	err := repo.packageList.ForEach(func(p *Package) error {
+		i++
+		if progress != nil {
+			progress.SetBar(i)
+		}
+		// download process might have updated checksums
+		p.UpdateFiles(p.Files())
+		return collectionFactory.PackageCollection().Update(p)
+	})
+
+	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+
+	if progress != nil {
+		progress.ShutdownBar()
+	}
+
+	repo.packageList = nil
+
+	return err
 }
 
 // Encode does msgpack encoding of RemoteRepo
@@ -563,7 +602,7 @@ func (repo *RemoteRepo) Decode(input []byte) error {
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "codec.decoder: readContainerLen: Unrecognized descriptor byte: hex: 80") {
 			// probably it is broken DB from go < 1.2, try decoding w/o time.Time
-			var repo11 struct {
+			var repo11 struct { // nolint: maligned
 				UUID             string
 				Name             string
 				ArchiveRoot      string

deb/remote_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -27,11 +28,11 @@ func (n *NullVerifier) InitKeyring() error {
 func (n *NullVerifier) AddKeyring(keyring string) {
 }
 
-func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) error {
+func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader, hint bool) error {
 	return nil
 }
 
-func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*utils.GpgKeyInfo, error) {
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*pgp.KeyInfo, error) {
 	return nil, nil
 }
 
@@ -81,6 +82,7 @@ type RemoteRepoSuite struct {
 	db                database.Storage
 	collectionFactory *CollectionFactory
 	packagePool       aptly.PackagePool
+	cs                aptly.ChecksumStorage
 }
 
 var _ = Suite(&RemoteRepoSuite{})
@@ -90,9 +92,10 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collectionFactory = NewCollectionFactory(s.db)
-	s.packagePool = files.NewPackagePool(c.MkDir())
+	s.packagePool = files.NewPackagePool(c.MkDir(), false)
+	s.cs = files.NewMockChecksumStorage()
 	s.SetUpPackages()
 	s.progress.Start()
 }
@@ -155,24 +158,30 @@ func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
 	c.Assert(s.flat.ReleaseURL("Release").String(), Equals, "http://repos.express42.com/virool/precise/Release")
 }
 
-func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
+func (s *RemoteRepoSuite) TestIndexesRootURL(c *C) {
-	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
+	c.Assert(s.repo.IndexesRootURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/")
+
+	c.Assert(s.flat.IndexesRootURL().String(), Equals, "http://repos.express42.com/virool/precise/")
 }
 
-func (s *RemoteRepoSuite) TestUdebURL(c *C) {
+func (s *RemoteRepoSuite) TestBinaryPath(c *C) {
-	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+	c.Assert(s.repo.BinaryPath("main", "amd64"), Equals, "main/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
+func (s *RemoteRepoSuite) TestUdebPath(c *C) {
-	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
+	c.Assert(s.repo.UdebPath("main", "amd64"), Equals, "main/debian-installer/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestFlatBinaryURL(c *C) {
+func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
-	c.Assert(s.flat.FlatBinaryURL().String(), Equals, "http://repos.express42.com/virool/precise/Packages")
+	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
-func (s *RemoteRepoSuite) TestFlatSourcesURL(c *C) {
+func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
-	c.Assert(s.flat.FlatSourcesURL().String(), Equals, "http://repos.express42.com/virool/precise/Sources")
+	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
+}
+
+func (s *RemoteRepoSuite) TestFlatSourcesPath(c *C) {
+	c.Assert(s.flat.FlatSourcesPath(), Equals, "Sources")
 }
 
 func (s *RemoteRepoSuite) TestPackageURL(c *C) {
@@ -266,18 +275,62 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
@@ -298,13 +351,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -312,7 +366,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
@@ -323,6 +377,56 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
@@ -340,18 +444,64 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
@@ -375,13 +525,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -389,7 +540,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
@@ -401,6 +552,58 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 type RemoteRepoCollectionSuite struct {
@@ -412,7 +615,7 @@ type RemoteRepoCollectionSuite struct {
 var _ = Suite(&RemoteRepoCollectionSuite{})
 
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
 	s.SetUpPackages()
 }
@@ -422,14 +625,14 @@ func (s *RemoteRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("yandex")
+	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("yandex")
+	r, err := s.collection.ByName("yandex")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -440,13 +643,13 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/snapshot.go

@@ -31,6 +31,10 @@ type Snapshot struct {
 	// Description of how snapshot was created
 	Description string
 
+	Origin               string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
+
 	packageRefs *PackageRefList
 }
 
@@ -41,31 +45,36 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 	}
 
 	return &Snapshot{
-		UUID:        uuid.New(),
+		UUID:                 uuid.New(),
-		Name:        name,
+		Name:                 name,
-		CreatedAt:   time.Now(),
+		CreatedAt:            time.Now(),
-		SourceKind:  "repo",
+		SourceKind:           SourceRemoteRepo,
-		SourceIDs:   []string{repo.UUID},
+		SourceIDs:            []string{repo.UUID},
-		Description: fmt.Sprintf("Snapshot from mirror %s", repo),
+		Description:          fmt.Sprintf("Snapshot from mirror %s", repo),
-		packageRefs: repo.packageRefs,
+		Origin:               repo.Meta["Origin"],
+		NotAutomatic:         repo.Meta["NotAutomatic"],
+		ButAutomaticUpgrades: repo.Meta["ButAutomaticUpgrades"],
+		packageRefs:          repo.packageRefs,
 	}, nil
 }
 
 // NewSnapshotFromLocalRepo creates snapshot from current state of local repository
 func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
-	if repo.packageRefs == nil {
+	snap := &Snapshot{
-		return nil, errors.New("local repo doesn't have packages")
-	}
-
-	return &Snapshot{
 		UUID:        uuid.New(),
 		Name:        name,
 		CreatedAt:   time.Now(),
-		SourceKind:  "local",
+		SourceKind:  SourceLocalRepo,
 		SourceIDs:   []string{repo.UUID},
 		Description: fmt.Sprintf("Snapshot from local repo %s", repo),
 		packageRefs: repo.packageRefs,
-	}, nil
+	}
+
+	if snap.packageRefs == nil {
+		snap.packageRefs = NewPackageRefList()
+	}
+
+	return snap, nil
 }
 
 // NewSnapshotFromPackageList creates snapshot from PackageList
@@ -255,7 +264,7 @@ func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Sn
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "repo" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}
@@ -267,7 +276,7 @@ func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snap
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "local" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}

deb/snapshot_test.go

@@ -26,7 +26,7 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
-	c.Check(snapshot.SourceKind, Equals, "repo")
+	c.Check(snapshot.SourceKind, Equals, SourceRemoteRepo)
 	c.Check(snapshot.SourceIDs, DeepEquals, []string{s.repo.UUID})
 
 	s.repo.packageRefs = nil
@@ -37,11 +37,17 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 func (s *SnapshotSuite) TestNewSnapshotFromLocalRepo(c *C) {
 	localRepo := NewLocalRepo("lala", "hoorah!")
 
-	_, err := NewSnapshotFromLocalRepo("snap2", localRepo)
+	snapshot, err := NewSnapshotFromLocalRepo("snap2", localRepo)
-	c.Check(err, ErrorMatches, "local repo doesn't have packages")
+	c.Assert(err, IsNil)
+	c.Check(snapshot.Name, Equals, "snap2")
+	c.Check(snapshot.NumPackages(), Equals, 0)
+	c.Check(snapshot.RefList().Len(), Equals, 0)
+	c.Check(snapshot.SourceKind, Equals, "local")
+	c.Check(snapshot.SourceIDs, DeepEquals, []string{localRepo.UUID})
 
 	localRepo.UpdateRefList(s.reflist)
-	snapshot, _ := NewSnapshotFromLocalRepo("snap1", localRepo)
+	snapshot, err = NewSnapshotFromLocalRepo("snap1", localRepo)
+	c.Assert(err, IsNil)
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
@@ -106,7 +112,7 @@ type SnapshotCollectionSuite struct {
 var _ = Suite(&SnapshotCollectionSuite{})
 
 func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
@@ -132,7 +138,7 @@ func (s *SnapshotCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
-	snapshot, err := s.collection.ByName("snap1")
+	_, err := s.collection.ByName("snap1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.snapshot1), IsNil)
@@ -140,7 +146,7 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 
 	c.Assert(s.collection.Add(s.snapshot2), IsNil)
 
-	snapshot, err = s.collection.ByName("snap1")
+	snapshot, err := s.collection.ByName("snap1")
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.String(), Equals, s.snapshot1.String())
 

deb/uploaders.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/DisposaBoy/JsonConfigReader"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -85,7 +86,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			deny := u.ExpandGroups(rule.Deny)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range deny {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return fmt.Errorf("denied according to rule: %s", rule)
 					}
 				}
@@ -94,7 +95,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			allow := u.ExpandGroups(rule.Allow)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range allow {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return nil
 					}
 				}

deb/uploaders_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	. "gopkg.in/check.v1"
 )
 
@@ -58,24 +58,24 @@ func (s *UploadersSuite) TestIsAllowed(c *C) {
 	}
 
 	// no keys - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// no rule - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// first rule: allow anyone do stuff with calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
 
 	// second rule: nobody is allowed to do stuff with never-calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"\\*\"\\]}")
 
 	// third rule: anyone from the group or explicit key
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
 
 	// fourth rule: some are not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"45678901\",\"12345678\"\\]}")
 }

deb/version.go

@@ -262,6 +262,13 @@ func ParseDependency(dep string) (d Dependency, err error) {
 	}
 
 	d.Pkg = strings.TrimSpace(dep[0:i])
+	if strings.ContainsRune(d.Pkg, ':') {
+		parts := strings.SplitN(d.Pkg, ":", 2)
+		d.Pkg, d.Architecture = parts[0], parts[1]
+		if d.Architecture == "any" {
+			d.Architecture = ""
+		}
+	}
 
 	rel := ""
 	if dep[i+1] == '>' || dep[i+1] == '<' || dep[i+1] == '=' {

deb/version_test.go

@@ -164,6 +164,20 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 	c.Check(d.Version, Equals, "1.6")
 	c.Check(d.Architecture, Equals, "i386")
 
+	d, e = ParseDependency("python:any (>= 2.7~)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "python")
+	c.Check(d.Relation, Equals, VersionGreaterOrEqual)
+	c.Check(d.Version, Equals, "2.7~")
+	c.Check(d.Architecture, Equals, "")
+
+	d, e = ParseDependency("python:amd64 (>= 2.7~)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "python")
+	c.Check(d.Relation, Equals, VersionGreaterOrEqual)
+	c.Check(d.Version, Equals, "2.7~")
+	c.Check(d.Architecture, Equals, "amd64")
+
 	d, e = ParseDependency("dpkg{i386}")
 	c.Check(e, IsNil)
 	c.Check(d.Pkg, Equals, "dpkg")

files/mocks.go

@@ -0,0 +1,36 @@
+package files
+
+import (
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+)
+
+type mockChecksumStorage struct {
+	store map[string]utils.ChecksumInfo
+}
+
+// NewMockChecksumStorage creates aptly.ChecksumStorage for tests
+func NewMockChecksumStorage() aptly.ChecksumStorage {
+	return &mockChecksumStorage{
+		store: make(map[string]utils.ChecksumInfo),
+	}
+}
+
+func (st *mockChecksumStorage) Get(path string) (*utils.ChecksumInfo, error) {
+	c, ok := st.store[path]
+	if !ok {
+		return nil, nil
+	}
+
+	return &c, nil
+}
+
+func (st *mockChecksumStorage) Update(path string, c *utils.ChecksumInfo) error {
+	st.store[path] = *c
+	return nil
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &mockChecksumStorage{}
+)

files/package_pool.go

@@ -7,33 +7,51 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"syscall"
+
+	"github.com/smira/go-uuid/uuid"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // PackagePool is deduplicated storage of package files on filesystem
 type PackagePool struct {
 	sync.Mutex
-	rootPath string
+
+	rootPath           string
+	supportLegacyPaths bool
 }
 
 // Check interface
 var (
-	_ aptly.PackagePool = (*PackagePool)(nil)
+	_ aptly.PackagePool      = (*PackagePool)(nil)
+	_ aptly.LocalPackagePool = (*PackagePool)(nil)
 )
 
 // NewPackagePool creates new instance of PackagePool which specified root
-func NewPackagePool(root string) *PackagePool {
+func NewPackagePool(root string, supportLegacyPaths bool) *PackagePool {
-	return &PackagePool{rootPath: filepath.Join(root, "pool")}
+	rootPath := filepath.Join(root, "pool")
+	rootPath, err := filepath.Abs(rootPath)
+	if err != nil {
+		panic(err)
+	}
+
+	return &PackagePool{
+		rootPath:           rootPath,
+		supportLegacyPaths: supportLegacyPaths,
+	}
 }
 
-// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+// LegacyPath returns path relative to pool's root for pre-1.1 aptly (based on MD5)
-func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
+func (pool *PackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
 	filename = filepath.Base(filename)
 	if filename == "." || filename == "/" {
 		return "", fmt.Errorf("filename %s is invalid", filename)
 	}
 
+	hashMD5 := checksums.MD5
+
 	if len(hashMD5) < 4 {
 		return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
 	}
@@ -41,14 +59,21 @@ func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string,
 	return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
 }
 
-// Path returns full path to package file in pool given any name and hash of file contents
+// buildPoolPath generates pool path based on file checksum
-func (pool *PackagePool) Path(filename string, hashMD5 string) (string, error) {
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
-	relative, err := pool.RelativePath(filename, hashMD5)
+	filename = filepath.Base(filename)
-	if err != nil {
+	if filename == "." || filename == "/" {
-		return "", err
+		return "", fmt.Errorf("filename %s is invalid", filename)
 	}
 
-	return filepath.Join(pool.rootPath, relative), nil
+	hash := checksums.SHA256
+
+	if len(hash) < 4 {
+		// this should never happen in real life
+		return "", fmt.Errorf("unable to compute pool location for filename %v, SHA256 is missing", filename)
+	}
+
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename), nil
 }
 
 // FilepathList returns file paths of all the files in the pool
@@ -113,57 +138,276 @@ func (pool *PackagePool) Remove(path string) (size int64, err error) {
 	return info.Size(), err
 }
 
+func (pool *PackagePool) ensureChecksums(poolPath, fullPoolPath string, checksumStorage aptly.ChecksumStorage) (targetChecksums *utils.ChecksumInfo, err error) {
+	targetChecksums, err = checksumStorage.Get(poolPath)
+	if err != nil {
+		return
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForFile(fullPoolPath)
+		if err != nil {
+			return
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+	}
+
+	return
+}
+
+// Verify checks whether file exists in the pool and fills back checksum info
+//
+// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	possiblePoolPaths := []string{}
+
+	if poolPath != "" {
+		possiblePoolPaths = append(possiblePoolPaths, poolPath)
+	} else {
+		// try to guess
+		if checksums.SHA256 != "" {
+			modernPath, err := pool.buildPoolPath(basename, checksums)
+			if err != nil {
+				return "", false, err
+			}
+			possiblePoolPaths = append(possiblePoolPaths, modernPath)
+		}
+
+		if pool.supportLegacyPaths && checksums.MD5 != "" {
+			legacyPath, err := pool.LegacyPath(basename, checksums)
+			if err != nil {
+				return "", false, err
+			}
+			possiblePoolPaths = append(possiblePoolPaths, legacyPath)
+		}
+	}
+
+	for _, path := range possiblePoolPaths {
+		fullPoolPath := filepath.Join(pool.rootPath, path)
+
+		targetInfo, err := os.Stat(fullPoolPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				// unable to stat target location?
+				return "", false, err
+			}
+			// doesn't exist, skip it
+			continue
+		}
+
+		if targetInfo.Size() != checksums.Size {
+			// oops, wrong file?
+			continue
+		}
+
+		var targetChecksums *utils.ChecksumInfo
+		targetChecksums, err = pool.ensureChecksums(path, fullPoolPath, checksumStorage)
+
+		if err != nil {
+			return "", false, err
+		}
+
+		if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+			checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+			// wrong file?
+			return "", false, nil
+		}
+
+		// fill back checksums
+		*checksums = *targetChecksums
+		return path, true, nil
+	}
+
+	return "", false, nil
+}
+
 // Import copies file into package pool
-func (pool *PackagePool) Import(path string, hashMD5 string) error {
+//
+// - srcPath is full path to source file as it is now
+// - basename is desired human-readable name (canonical filename)
+// - checksums are used to calculate file placement
+// - move indicates whether srcPath can be removed
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, checksumStorage aptly.ChecksumStorage) (string, error) {
 	pool.Lock()
 	defer pool.Unlock()
 
-	source, err := os.Open(path)
+	source, err := os.Open(srcPath)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer source.Close()
 
 	sourceInfo, err := source.Stat()
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	poolPath, err := pool.Path(path, hashMD5)
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.Size != sourceInfo.Size() {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		*checksums, err = utils.ChecksumsForFile(srcPath)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	// build target path
+	poolPath, err := pool.buildPoolPath(basename, checksums)
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	targetInfo, err := os.Stat(poolPath)
+	fullPoolPath := filepath.Join(pool.rootPath, poolPath)
+
+	targetInfo, err := os.Stat(fullPoolPath)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			// unable to stat target location?
-			return err
+			return "", err
 		}
 	} else {
-		// target already exists
+		// target already exists and same size
-		if targetInfo.Size() != sourceInfo.Size() {
+		if targetInfo.Size() == sourceInfo.Size() {
-			// trying to overwrite file?
+			var targetChecksums *utils.ChecksumInfo
-			return fmt.Errorf("unable to import into pool: file %s already exists", poolPath)
+
+			targetChecksums, err = pool.ensureChecksums(poolPath, fullPoolPath, checksumStorage)
+			if err != nil {
+				return "", err
+			}
+
+			*checksums = *targetChecksums
+			return poolPath, nil
 		}
 
-		// assume that target is already there
+		// trying to overwrite file?
-		return nil
+		return "", fmt.Errorf("unable to import into pool: file %s already exists", fullPoolPath)
+	}
+
+	if pool.supportLegacyPaths {
+		// file doesn't exist at new location, check legacy location
+		var (
+			legacyTargetInfo           os.FileInfo
+			legacyPath, legacyFullPath string
+		)
+
+		legacyPath, err = pool.LegacyPath(basename, checksums)
+		if err != nil {
+			return "", err
+		}
+		legacyFullPath = filepath.Join(pool.rootPath, legacyPath)
+
+		legacyTargetInfo, err = os.Stat(legacyFullPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return "", err
+			}
+		} else {
+			// legacy file exists
+			if legacyTargetInfo.Size() == sourceInfo.Size() {
+				// file exists at legacy path and it's same size, consider it's already in the pool
+				var targetChecksums *utils.ChecksumInfo
+
+				targetChecksums, err = pool.ensureChecksums(legacyPath, legacyFullPath, checksumStorage)
+				if err != nil {
+					return "", err
+				}
+
+				*checksums = *targetChecksums
+				return legacyPath, nil
+			}
+
+			// size is different, import at new path
+		}
 	}
 
 	// create subdirs as necessary
-	err = os.MkdirAll(filepath.Dir(poolPath), 0777)
+	poolDir := filepath.Dir(fullPoolPath)
+	err = os.MkdirAll(poolDir, 0777)
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	target, err := os.Create(poolPath)
+	// check if we can use hardlinks instead of copying/moving
+	poolDirInfo, err := os.Stat(poolDir)
 	if err != nil {
-		return err
+		return "", err
 	}
-	defer target.Close()
 
-	_, err = io.Copy(target, source)
+	if poolDirInfo.Sys().(*syscall.Stat_t).Dev == sourceInfo.Sys().(*syscall.Stat_t).Dev {
+		// same filesystem, try to use hardlink
+		err = os.Link(srcPath, fullPoolPath)
+	} else {
+		err = os.ErrInvalid
+	}
 
-	return err
+	if err != nil {
+		// different filesystems or failed hardlink, fallback to copy
+		var target *os.File
+		target, err = os.Create(fullPoolPath)
+		if err != nil {
+			return "", err
+		}
+		defer target.Close()
+
+		_, err = io.Copy(target, source)
+
+		if err == nil {
+			err = target.Close()
+		}
+	}
+
+	if err == nil {
+		if !checksums.Complete() {
+			// need full checksums here
+			*checksums, err = utils.ChecksumsForFile(srcPath)
+			if err != nil {
+				return "", err
+			}
+		}
+
+		err = checksumStorage.Update(poolPath, checksums)
+	}
+
+	if err == nil && move {
+		err = os.Remove(srcPath)
+	}
+
+	return poolPath, err
+}
+
+// Open returns io.ReadCloser to access the file
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	return os.Open(filepath.Join(pool.rootPath, path))
+}
+
+// Stat returns Unix stat(2) info
+func (pool *PackagePool) Stat(path string) (os.FileInfo, error) {
+	return os.Stat(filepath.Join(pool.rootPath, path))
+}
+
+// Link generates hardlink to destination path
+func (pool *PackagePool) Link(path, dstPath string) error {
+	return os.Link(filepath.Join(pool.rootPath, path), dstPath)
+}
+
+// Symlink generates symlink to destination path
+func (pool *PackagePool) Symlink(path, dstPath string) error {
+	return os.Symlink(filepath.Join(pool.rootPath, path), dstPath)
+}
+
+// FullPath generates full path to the file in pool
+//
+// Please use with care: it's not supposed to be used to access files
+func (pool *PackagePool) FullPath(path string) string {
+	return filepath.Join(pool.rootPath, path)
+}
+
+// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+func (pool *PackagePool) GenerateTempPath(filename string) (string, error) {
+	random := uuid.NewRandom().String()
+
+	return filepath.Join(pool.rootPath, random[0:2], random[2:4], random[4:]+filename), nil
 }

files/package_pool_test.go

@@ -1,47 +1,51 @@
 package files
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"syscall"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
 
 type PackagePoolSuite struct {
-	pool *PackagePool
+	pool     *PackagePool
+	checksum utils.ChecksumInfo
+	debFile  string
+	cs       aptly.ChecksumStorage
 }
 
 var _ = Suite(&PackagePoolSuite{})
 
 func (s *PackagePoolSuite) SetUpTest(c *C) {
-	s.pool = NewPackagePool(c.MkDir())
+	s.pool = NewPackagePool(c.MkDir(), true)
-
+	s.checksum = utils.ChecksumInfo{
+		MD5: "0035d7822b2f8f0ec4013f270fd650c2",
+	}
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = NewMockChecksumStorage()
 }
 
-func (s *PackagePoolSuite) TestRelativePath(c *C) {
+func (s *PackagePoolSuite) TestLegacyPath(c *C) {
-	path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
+	path, err := s.pool.LegacyPath("a/b/package.deb", &s.checksum)
 	c.Assert(err, IsNil)
-	c.Assert(path, Equals, "91/b1/package.deb")
+	c.Assert(path, Equals, "00/35/package.deb")
 
-	_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
+	_, err = s.pool.LegacyPath("/", &s.checksum)
 	c.Assert(err, ErrorMatches, ".*is invalid")
-	_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
+	_, err = s.pool.LegacyPath("", &s.checksum)
 	c.Assert(err, ErrorMatches, ".*is invalid")
-	_, err = s.pool.RelativePath("a/b/package.deb", "9")
+	_, err = s.pool.LegacyPath("a/b/package.deb", &utils.ChecksumInfo{MD5: "9"})
 	c.Assert(err, ErrorMatches, ".*MD5 is missing")
 }
 
-func (s *PackagePoolSuite) TestPath(c *C) {
-	path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, IsNil)
-	c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
-
-	_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, ErrorMatches, ".*is invalid")
-}
-
 func (s *PackagePoolSuite) TestFilepathList(c *C) {
 	list, err := s.pool.FilepathList(nil)
 	c.Check(err, IsNil)
@@ -88,33 +92,264 @@ func (s *PackagePoolSuite) TestRemove(c *C) {
 }
 
 func (s *PackagePoolSuite) TestImportOk(c *C) {
-	_, _File, _, _ := runtime.Caller(0)
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
-	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
-
-	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
 	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(s.checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*mockChecksumStorage).store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
 
-	info, err := os.Stat(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	info, err := s.pool.Stat(path)
-	c.Check(err, IsNil)
+	c.Assert(err, IsNil)
 	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 1, Equals, true)
+
+	// import as different name
+	path, err = s.pool.Import(s.debFile, "some.deb", &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*mockChecksumStorage).store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
 
 	// double import, should be ok
-	err = s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
 	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*mockChecksumStorage).store, path)
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with checksums already filled in
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, "other.deb", &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestImportLegacy(c *C) {
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "00", "35"), 0755)
+	err := utils.CopyFile(s.debFile, filepath.Join(s.pool.rootPath, "00", "35", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	c.Assert(err, IsNil)
+
+	s.checksum.Size = 2738
+	var path string
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerifyLegacy(c *C) {
+	s.checksum.Size = 2738
+	// file doesn't exist yet
+	path, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(path, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "00", "35"), 0755)
+	err = utils.CopyFile(s.debFile, filepath.Join(s.pool.rootPath, "00", "35", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	c.Assert(err, IsNil)
+
+	// check existence (and fills back checksum)
+	path, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with missing checksum
+	s.checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	ck := utils.ChecksumInfo{
+		Size: s.checksum.Size,
+	}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &ck, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(ck.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ck.SHA256 = "abc"
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &ck, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksum and no info in checksum storage
+	delete(s.cs.(*mockChecksumStorage).store, path)
+	s.checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	s.checksum.Size = 13455
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportMove(c *C) {
+	tmpDir := c.MkDir()
+	tmpPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+
+	dst, err := os.Create(tmpPath)
+	c.Assert(err, IsNil)
+
+	src, err := os.Open(s.debFile)
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(dst, src)
+	c.Assert(err, IsNil)
+
+	c.Assert(dst.Close(), IsNil)
+	c.Assert(src.Close(), IsNil)
+
+	path, err := s.pool.Import(tmpPath, filepath.Base(tmpPath), &s.checksum, true, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	info, err := s.pool.Stat(path)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(int(info.Sys().(*syscall.Stat_t).Nlink), Equals, 1)
 }
 
 func (s *PackagePoolSuite) TestImportNotExist(c *C) {
-	err := s.pool.Import("no-such-file", "91b1a1480b90b9e269ca44d897b12575")
+	_, err := s.pool.Import("no-such-file", "a.deb", &s.checksum, false, s.cs)
 	c.Check(err, ErrorMatches, ".*no such file or directory")
 }
 
 func (s *PackagePoolSuite) TestImportOverwrite(c *C) {
-	_, _File, _, _ := runtime.Caller(0)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "c7", "6b"), 0755)
-	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "c7", "6b", "4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
 
-	os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755)
+	_, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
-	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
-
-	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
 	c.Check(err, ErrorMatches, "unable to import into pool.*")
 }
+
+func (s *PackagePoolSuite) TestStat(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	info, err := s.pool.Stat(path)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+
+	_, err = s.pool.Stat("do/es/ntexist")
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *PackagePoolSuite) TestLink(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	tmpDir := c.MkDir()
+	dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+	c.Check(s.pool.Link(path, dstPath), IsNil)
+
+	info, err := os.Stat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true)
+}
+
+func (s *PackagePoolSuite) TestSymlink(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	tmpDir := c.MkDir()
+	dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+	c.Check(s.pool.Symlink(path, dstPath), IsNil)
+
+	info, err := os.Stat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true)
+
+	info, err = os.Lstat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(int(info.Sys().(*syscall.Stat_t).Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
+}
+
+func (s *PackagePoolSuite) TestGenerateRandomPath(c *C) {
+	path, err := s.pool.GenerateTempPath("a.deb")
+	c.Check(err, IsNil)
+
+	c.Check(path, Matches, ".+/[0-9a-f][0-9a-f]/[0-9a-f][0-9a-f]/[0-9a-f-]+a\\.deb")
+}

files/public.go

@@ -5,25 +5,62 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
 type PublishedStorage struct {
-	rootPath string
+	rootPath     string
+	linkMethod   uint
+	verifyMethod uint
 }
 
 // Check interfaces
 var (
-	_ aptly.PublishedStorage      = (*PublishedStorage)(nil)
+	_ aptly.PublishedStorage           = (*PublishedStorage)(nil)
-	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.FileSystemPublishedStorage = (*PublishedStorage)(nil)
+)
+
+// Constants defining the type of creating links
+const (
+	LinkMethodHardLink uint = iota
+	LinkMethodSymLink
+	LinkMethodCopy
+)
+
+// Constants defining the type of file verification for LinkMethodCopy
+const (
+	VerificationMethodChecksum uint = iota
+	VerificationMethodFileSize
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
-func NewPublishedStorage(root string) *PublishedStorage {
+func NewPublishedStorage(root string, linkMethod string, verifyMethod string) *PublishedStorage {
-	return &PublishedStorage{rootPath: filepath.Join(root, "public")}
+	// Ensure linkMethod is one of 'hardlink', 'symlink', 'copy'
+	var verifiedLinkMethod uint
+
+	if strings.EqualFold(linkMethod, "copy") {
+		verifiedLinkMethod = LinkMethodCopy
+	} else if strings.EqualFold(linkMethod, "symlink") {
+		verifiedLinkMethod = LinkMethodSymLink
+	} else {
+		verifiedLinkMethod = LinkMethodHardLink
+	}
+
+	var verifiedVerifyMethod uint
+
+	if strings.EqualFold(verifyMethod, "size") {
+		verifiedVerifyMethod = VerificationMethodFileSize
+	} else {
+		verifiedVerifyMethod = VerificationMethodChecksum
+	}
+
+	return &PublishedStorage{rootPath: root, linkMethod: verifiedLinkMethod,
+		verifyMethod: verifiedVerifyMethod}
 }
 
 // PublicPath returns root of public part
@@ -77,15 +114,12 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 //
 // publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
-// sourcePath is filepath to package file in package pool
+// sourcePath is a relative path to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
-	sourcePath, sourceMD5 string, force bool) error {
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
-	// verify that package pool is local pool is filesystem pool
-	_ = sourcePool.(*PackagePool)
 
-	baseName := filepath.Base(sourcePath)
 	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
 
 	err := os.MkdirAll(poolPath, 0777)
@@ -98,18 +132,42 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
 	if err == nil {
 		// already exists, check source file
-		srcStat, err = os.Stat(sourcePath)
+		srcStat, err = sourcePool.Stat(sourcePath)
 		if err != nil {
 			// source file doesn't exist? problem!
 			return err
 		}
 
-		srcSys := srcStat.Sys().(*syscall.Stat_t)
+		if storage.linkMethod == LinkMethodCopy {
-		dstSys := dstStat.Sys().(*syscall.Stat_t)
+			if storage.verifyMethod == VerificationMethodFileSize {
+				// if source and destination have the same size, no need to copy
+				if srcStat.Size() == dstStat.Size() {
+					return nil
+				}
+			} else {
+				// if source and destination have the same checksums, no need to copy
+				var dstMD5 string
+				dstMD5, err = utils.MD5ChecksumForFile(filepath.Join(poolPath, baseName))
 
-		// source and destination inodes match, no need to link
+				if err != nil {
-		if srcSys.Ino == dstSys.Ino {
+					return err
-			return nil
+				}
+
+				if dstMD5 == sourceChecksums.MD5 {
+					return nil
+				}
+			}
+		} else {
+			srcSys := srcStat.Sys().(*syscall.Stat_t)
+			dstSys := dstStat.Sys().(*syscall.Stat_t)
+
+			// if source and destination inodes match, no need to link
+
+			// Symlink can point to different filesystem with identical inodes
+			// so we have to check the device as well.
+			if srcSys.Ino == dstSys.Ino && srcSys.Dev == dstSys.Dev {
+				return nil
+			}
 		}
 
 		// source and destination have different inodes, if !forced, this is fatal error
@@ -124,8 +182,42 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		}
 	}
 
-	// destination doesn't exist (or forced), create link
+	// destination doesn't exist (or forced), create link or copy
-	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
+	if storage.linkMethod == LinkMethodCopy {
+		var r aptly.ReadSeekerCloser
+		r, err = sourcePool.Open(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		var dst *os.File
+		dst, err = os.Create(filepath.Join(poolPath, baseName))
+		if err != nil {
+			r.Close()
+			return err
+		}
+
+		_, err = io.Copy(dst, r)
+		if err != nil {
+			r.Close()
+			dst.Close()
+			return err
+		}
+
+		err = r.Close()
+		if err != nil {
+			dst.Close()
+			return err
+		}
+
+		err = dst.Close()
+	} else if storage.linkMethod == LinkMethodSymLink {
+		err = sourcePool.(aptly.LocalPackagePool).Symlink(sourcePath, filepath.Join(poolPath, baseName))
+	} else {
+		err = sourcePool.(aptly.LocalPackagePool).Link(sourcePath, filepath.Join(poolPath, baseName))
+	}
+
+	return err
 }
 
 // Filelist returns list of files under prefix
@@ -155,3 +247,27 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 	return os.Rename(filepath.Join(storage.rootPath, oldName), filepath.Join(storage.rootPath, newName))
 }
+
+// SymLink creates a symbolic link, which can be read with ReadLink
+func (storage *PublishedStorage) SymLink(src string, dst string) error {
+	return os.Symlink(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// HardLink creates a hardlink of a file
+func (storage *PublishedStorage) HardLink(src string, dst string) error {
+	return os.Link(filepath.Join(storage.rootPath, src), filepath.Join(storage.rootPath, dst))
+}
+
+// FileExists returns true if path exists
+func (storage *PublishedStorage) FileExists(path string) (bool, error) {
+	if _, err := os.Lstat(filepath.Join(storage.rootPath, path)); os.IsNotExist(err) {
+		return false, nil
+	}
+
+	return true, nil
+}
+
+// ReadLink returns the symbolic link pointed to by path
+func (storage *PublishedStorage) ReadLink(path string) (string, error) {
+	return os.Readlink(path)
+}

files/public_test.go

@@ -6,23 +6,49 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type PublishedStorageSuite struct {
-	root    string
+	root            string
-	storage *PublishedStorage
+	storage         *PublishedStorage
+	storageSymlink  *PublishedStorage
+	storageCopy     *PublishedStorage
+	storageCopySize *PublishedStorage
+	cs              aptly.ChecksumStorage
 }
 
 var _ = Suite(&PublishedStorageSuite{})
 
 func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	s.root = c.MkDir()
-	s.storage = NewPublishedStorage(s.root)
+	s.storage = NewPublishedStorage(filepath.Join(s.root, "public"), "", "")
+	s.storageSymlink = NewPublishedStorage(filepath.Join(s.root, "public_symlink"), "symlink", "")
+	s.storageCopy = NewPublishedStorage(filepath.Join(s.root, "public_copy"), "copy", "")
+	s.storageCopySize = NewPublishedStorage(filepath.Join(s.root, "public_copysize"), "copy", "size")
+	s.cs = NewMockChecksumStorage()
+}
+
+func (s *PublishedStorageSuite) TestLinkMethodField(c *C) {
+	c.Assert(s.storage.linkMethod, Equals, LinkMethodHardLink)
+	c.Assert(s.storageSymlink.linkMethod, Equals, LinkMethodSymLink)
+	c.Assert(s.storageCopy.linkMethod, Equals, LinkMethodCopy)
+	c.Assert(s.storageCopySize.linkMethod, Equals, LinkMethodCopy)
+}
+
+func (s *PublishedStorageSuite) TestVerifyMethodField(c *C) {
+	c.Assert(s.storageCopy.verifyMethod, Equals, VerificationMethodChecksum)
+	c.Assert(s.storageCopySize.verifyMethod, Equals, VerificationMethodFileSize)
 }
 
 func (s *PublishedStorageSuite) TestPublicPath(c *C) {
 	c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
+	c.Assert(s.storageSymlink.PublicPath(), Equals, filepath.Join(s.root, "public_symlink"))
+	c.Assert(s.storageCopy.PublicPath(), Equals, filepath.Join(s.root, "public_copy"))
+	c.Assert(s.storageCopySize.PublicPath(), Equals, filepath.Join(s.root, "public_copysize"))
 }
 
 func (s *PublishedStorageSuite) TestMkDir(c *C) {
@@ -33,7 +59,7 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *PublishedStorageSuite) TesPutFile(c *C) {
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
@@ -77,6 +103,48 @@ func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestFileExists(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, false)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	exists, _ = s.storage.FileExists("ppa/dists/squeeze/Release")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestSymLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.SymLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
+func (s *PublishedStorageSuite) TestHardLink(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	err = s.storage.HardLink("ppa/dists/squeeze/Release", "ppa/dists/squeeze/InRelease")
+	c.Assert(err, IsNil)
+
+	exists, _ := s.storage.FileExists("ppa/dists/squeeze/InRelease")
+	c.Check(exists, Equals, true)
+}
+
 func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
@@ -85,6 +153,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	c.Assert(err, IsNil)
 
 	err = s.storage.RemoveDirs("ppa/dists/", nil)
+	c.Assert(err, IsNil)
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
@@ -99,6 +168,7 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	c.Assert(err, IsNil)
 
 	err = s.storage.Remove("ppa/dists/squeeze/Release")
+	c.Assert(err, IsNil)
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
@@ -116,78 +186,133 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		{ // package name regular
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			sourcePath:       "mars-invaders_1.03.deb",
 			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // lib-like filename
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/libmars-invaders_1.03.deb",
+			sourcePath:       "libmars-invaders_1.03.deb",
 			poolDirectory:    "libm/libmars-invaders",
 			expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
 		},
 		{ // duplicate link, shouldn't panic
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			sourcePath:       "mars-invaders_1.03.deb",
 			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // prefix & component
 			prefix:           "ppa",
 			component:        "contrib",
-			sourcePath:       "pool/01/ae/libmars-invaders_1.04.deb",
+			sourcePath:       "libmars-invaders_1.04.deb",
 			poolDirectory:    "libm/libmars-invaders",
 			expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
 		},
 	}
 
-	pool := NewPackagePool(s.root)
+	pool := NewPackagePool(s.root, false)
 
 	for _, t := range tests {
-		t.sourcePath = filepath.Join(s.root, t.sourcePath)
+		tmpPath := filepath.Join(c.MkDir(), t.sourcePath)
-
+		err := ioutil.WriteFile(tmpPath, []byte("Contents"), 0644)
-		err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
 		c.Assert(err, IsNil)
 
-		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
+		sourceChecksum, err := utils.ChecksumsForFile(tmpPath)
 		c.Assert(err, IsNil)
 
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "", false)
+		srcPoolPath, err := pool.Import(tmpPath, t.sourcePath, &utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}, false, s.cs)
+		c.Assert(err, IsNil)
+
+		// Test using hardlinks
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
 		c.Assert(err, IsNil)
 
 		info := st.Sys().(*syscall.Stat_t)
-		c.Check(int(info.Nlink), Equals, 2)
+		c.Check(int(info.Nlink), Equals, 3)
+
+		// Test using symlinks
+		err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		c.Assert(err, IsNil)
+
+		st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
+		c.Assert(err, IsNil)
+
+		info = st.Sys().(*syscall.Stat_t)
+		c.Check(int(info.Nlink), Equals, 1)
+		c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
+
+		// Test using copy with checksum verification
+		err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		c.Assert(err, IsNil)
+
+		st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
+		c.Assert(err, IsNil)
+
+		info = st.Sys().(*syscall.Stat_t)
+		c.Check(int(info.Nlink), Equals, 1)
+
+		// Test using copy with size verification
+		err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		c.Assert(err, IsNil)
+
+		st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))
+		c.Assert(err, IsNil)
+
+		info = st.Sys().(*syscall.Stat_t)
+		c.Check(int(info.Nlink), Equals, 1)
 	}
 
 	// test linking files to duplicate final name
-	sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
+	tmpPath := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
-	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
+	err := ioutil.WriteFile(tmpPath, []byte("cONTENTS"), 0644)
 	c.Assert(err, IsNil)
 
-	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+	sourceChecksum, err := utils.ChecksumsForFile(tmpPath)
 	c.Assert(err, IsNil)
 
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", false)
+	srcPoolPath, err := pool.Import(tmpPath, "mars-invaders_1.03.deb", &utils.ChecksumInfo{MD5: "02bcda7a1ce305a3b60af9d5733ac1d"}, true, s.cs)
+	c.Assert(err, IsNil)
+
+	st, err := pool.Stat(srcPoolPath)
+	c.Assert(err, IsNil)
+	nlinks := int(st.Sys().(*syscall.Stat_t).Nlink)
+
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
 	c.Check(err, ErrorMatches, ".*file already exists and is different")
 
-	st, err := os.Stat(sourcePath)
+	st, err = pool.Stat(srcPoolPath)
 	c.Assert(err, IsNil)
-
+	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks)
-	info := st.Sys().(*syscall.Stat_t)
-	c.Check(int(info.Nlink), Equals, 1)
 
 	// linking with force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", true)
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
 	c.Check(err, IsNil)
 
-	st, err = os.Stat(sourcePath)
+	st, err = pool.Stat(srcPoolPath)
 	c.Assert(err, IsNil)
+	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1)
 
-	info = st.Sys().(*syscall.Stat_t)
+	// Test using symlinks
-	c.Check(int(info.Nlink), Equals, 2)
+	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	c.Check(err, IsNil)
+
+	// Test using copy with checksum verification
+	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	c.Check(err, IsNil)
+
+	// Test using copy with size verification (this will NOT detect the difference)
+	err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, IsNil)
 }

http/compression.go

@@ -0,0 +1,96 @@
+package http
+
+import (
+	"compress/bzip2"
+	"compress/gzip"
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	xz "github.com/smira/go-xz"
+)
+
+// List of extensions + corresponding uncompression support
+var compressionMethods = []struct {
+	extenstion     string
+	transformation func(io.Reader) (io.Reader, error)
+}{
+	{
+		extenstion:     ".bz2",
+		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
+	},
+	{
+		extenstion:     ".gz",
+		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
+	},
+	{
+		extenstion:     ".xz",
+		transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
+	},
+	{
+		extenstion:     "",
+		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
+	},
+}
+
+// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
+// it finds existing file.
+func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
+	var err error
+
+	for _, method := range compressionMethods {
+		var file *os.File
+
+		tryPath := path + method.extenstion
+		foundChecksum := false
+
+		bestSuffix := ""
+
+		for suffix := range expectedChecksums {
+			if strings.HasSuffix(tryPath, suffix) {
+				foundChecksum = true
+				if len(suffix) > len(bestSuffix) {
+					bestSuffix = suffix
+				}
+			}
+		}
+
+		tryURL := baseURL.ResolveReference(&url.URL{Path: tryPath})
+
+		if foundChecksum {
+			expected := expectedChecksums[bestSuffix]
+			file, err = DownloadTempWithChecksum(ctx, downloader, tryURL.String(), &expected, ignoreMismatch, maxTries)
+		} else {
+			if !ignoreMismatch {
+				continue
+			}
+
+			file, err = DownloadTemp(ctx, downloader, tryURL.String())
+		}
+
+		if err != nil {
+			if err1, ok := err.(*Error); ok && (err1.Code == 404 || err1.Code == 403) {
+				continue
+			}
+			return nil, nil, err
+		}
+
+		var uncompressed io.Reader
+		uncompressed, err = method.transformation(file)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return uncompressed, file, err
+	}
+
+	if err == nil {
+		err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path}))
+	}
+	return nil, nil, err
+}

http/compression_test.go

@@ -0,0 +1,149 @@
+package http
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net/url"
+
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type CompressionSuite struct {
+	baseURL *url.URL
+	ctx     context.Context
+}
+
+var _ = Suite(&CompressionSuite{})
+
+const (
+	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
+	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
+	xzData   = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
+	rawData  = "test"
+)
+
+func (s *CompressionSuite) SetUpTest(c *C) {
+	s.baseURL, _ = url.Parse("http://example.com/")
+	s.ctx = context.Background()
+}
+
+func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
+	var buf []byte
+
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: int64(len(bzipData))},
+		"file.gz":  {Size: int64(len(gzipData))},
+		"file.xz":  {Size: int64(len(xzData))},
+		"file":     {Size: int64(len(rawData))},
+	}
+
+	// bzip2 only available
+	buf = make([]byte, 4)
+	d := NewFakeDownloader()
+	d.ExpectResponse("http://example.com/file.bz2", bzipData)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 not available, but gz is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.gz", gzipData)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 & gzip not available, but xz is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.xz", xzData)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2, gzip & xz not available, but raw is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file", rawData)
+	r, file, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// gzip available, but broken
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.gz", "x")
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "unexpected EOF")
+	c.Assert(d.Empty(), Equals, true)
+}
+
+func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
+	var buf []byte
+
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2":          {Size: 1},
+		"subdir/file.bz2":   {Size: int64(len(bzipData))},
+		"otherdir/file.bz2": {Size: 1},
+	}
+
+	// longest suffix should be picked up
+	buf = make([]byte, 4)
+	d := NewFakeDownloader()
+	d.ExpectResponse("http://example.com/subdir/file.bz2", bzipData)
+	r, file, err := DownloadTryCompression(s.ctx, d, s.baseURL, "subdir/file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+}
+
+func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
+	d := NewFakeDownloader()
+	_, _, err := DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "unexpected request.*")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file", errors.New("403"))
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "403")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file", rawData)
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: 7},
+		"file.gz":  {Size: 7},
+		"file.xz":  {Size: 7},
+		"file":     {Size: 7},
+	}
+	_, _, err = DownloadTryCompression(s.ctx, d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, ErrorMatches, "checksums don't match.*")
+}

http/download.go

@@ -1,35 +1,24 @@
 package http
 
 import (
-	"compress/bzip2"
+	"context"
-	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/mxk/go-flowrate/flowrate"
+	"github.com/pkg/errors"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-ftp-protocol/protocol"
-	"github.com/smira/go-xz"
 )
 
-// Error is download error connected to HTTP code
-type Error struct {
-	Code int
-	URL  string
-}
-
-// Error
-func (e *Error) Error() string {
-	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
-}
-
 // Check interface
 var (
 	_ aptly.Downloader = (*downloaderImpl)(nil)
@@ -37,30 +26,14 @@ var (
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue     chan *downloadTask
-	stop      chan struct{}
-	stopped   chan struct{}
-	pause     chan struct{}
-	unpause   chan struct{}
 	progress  aptly.Progress
 	aggWriter io.Writer
-	threads   int
 	client    *http.Client
 }
 
-// downloadTask represents single item in queue
-type downloadTask struct {
-	url            string
-	destination    string
-	result         chan<- error
-	expected       utils.ChecksumInfo
-	ignoreMismatch bool
-	triesLeft      int
-}
-
 // NewDownloader creates new instance of Downloader which specified number
 // of threads and download limit in bytes/sec
-func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
+func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
 	transport := http.Transport{}
 	transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
 	transport.ResponseHeaderTimeout = 30 * time.Second
@@ -71,12 +44,6 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 	transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
 
 	downloader := &downloaderImpl{
-		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan struct{}, threads),
-		stopped:  make(chan struct{}, threads),
-		pause:    make(chan struct{}),
-		unpause:  make(chan struct{}),
-		threads:  threads,
 		progress: progress,
 		client: &http.Client{
 			Transport: &transport,
@@ -89,72 +56,43 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 		downloader.aggWriter = progress
 	}
 
-	for i := 0; i < downloader.threads; i++ {
-		go downloader.process()
-	}
-
 	return downloader
 }
 
-// Shutdown stops downloader after current tasks are finished,
-// but doesn't process rest of queue
-func (downloader *downloaderImpl) Shutdown() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- struct{}{}
-	}
-
-	for i := 0; i < downloader.threads; i++ {
-		<-downloader.stopped
-	}
-}
-
-// Abort stops downloader but doesn't wait for downloader to stop
-func (downloader *downloaderImpl) Abort() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- struct{}{}
-	}
-}
-
-// Pause pauses task processing
-func (downloader *downloaderImpl) Pause() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.pause <- struct{}{}
-	}
-}
-
-// Resume resumes task processing
-func (downloader *downloaderImpl) Resume() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.unpause <- struct{}{}
-	}
-}
-
 // GetProgress returns Progress object
 func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 	return downloader.progress
 }
 
 // Download starts new download task
-func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
+func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error {
-	downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false, 1)
+	return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1)
+}
+
+func retryableError(err error) bool {
+	switch err.(type) {
+	case net.Error:
+		return true
+	case *net.OpError:
+		return true
+	case syscall.Errno:
+		return true
+	}
+	return false
 }
 
 // DownloadWithChecksum starts new download task with checksum verification
-func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
+func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string,
-	expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
+	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
-	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
-}
 
-// handleTask processes single download task
+	downloader.progress.Printf("Downloading %s...\n", url)
-func (downloader *downloaderImpl) handleTask(task *downloadTask) {
-	downloader.progress.Printf("Downloading %s...\n", task.url)
 
-	req, err := http.NewRequest("GET", task.url, nil)
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return errors.Wrap(err, url)
-		return
 	}
 	req.Close = true
+	req = req.WithContext(ctx)
 
 	proxyURL, _ := downloader.client.Transport.(*http.Transport).Proxy(req)
 	if proxyURL == nil && (req.URL.Scheme == "http" || req.URL.Scheme == "https") {
@@ -163,64 +101,61 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	}
 
 	var temppath string
-	for task.triesLeft > 0 {
+	for maxTries > 0 {
+		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)
 
-		temppath, err = downloader.downloadTask(req, task)
+		if err != nil && retryableError(err) {
-
+			maxTries--
-		if err != nil {
-			task.triesLeft--
 		} else {
-			// successful download
+			// get out of the loop
 			break
 		}
 	}
 
 	// still an error after retrying, giving up
 	if err != nil {
-		task.result <- err
+		return err
-		return
 	}
 
-	err = os.Rename(temppath, task.destination)
+	err = os.Rename(temppath, destination)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return errors.Wrap(err, url)
-		return
 	}
 
-	task.result <- nil
+	return nil
 }
 
-func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
+func (downloader *downloaderImpl) download(req *http.Request, url, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) (string, error) {
 	resp, err := downloader.client.Do(req)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 	if resp.Body != nil {
 		defer resp.Body.Close()
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return "", &Error{Code: resp.StatusCode, URL: task.url}
+		return "", &Error{Code: resp.StatusCode, URL: url}
 	}
 
-	err = os.MkdirAll(filepath.Dir(task.destination), 0777)
+	err = os.MkdirAll(filepath.Dir(destination), 0777)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 
-	temppath := task.destination + ".down"
+	temppath := destination + ".down"
 
 	outfile, err := os.Create(temppath)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
 	writers := []io.Writer{outfile, downloader.aggWriter}
 
-	if task.expected.Size != -1 {
+	if expected != nil {
 		writers = append(writers, checksummer)
 	}
 
@@ -229,161 +164,36 @@ func (downloader *downloaderImpl) downloadTask(req *http.Request, task *download
 	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 
-	if task.expected.Size != -1 {
+	if expected != nil {
 		actual := checksummer.Sum()
 
-		if actual.Size != task.expected.Size {
+		if actual.Size != expected.Size {
-			err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
+			err = fmt.Errorf("%s: size check mismatch %d != %d", url, actual.Size, expected.Size)
-		} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
+		} else if expected.MD5 != "" && actual.MD5 != expected.MD5 {
-			err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
+			err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", url, actual.MD5, expected.MD5)
-		} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
+		} else if expected.SHA1 != "" && actual.SHA1 != expected.SHA1 {
-			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
+			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", url, actual.SHA1, expected.SHA1)
-		} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
+		} else if expected.SHA256 != "" && actual.SHA256 != expected.SHA256 {
-			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
+			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", url, actual.SHA256, expected.SHA256)
-		} else if task.expected.SHA512 != "" && actual.SHA512 != task.expected.SHA512 {
+		} else if expected.SHA512 != "" && actual.SHA512 != expected.SHA512 {
-			err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", task.url, actual.SHA512, task.expected.SHA512)
+			err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", url, actual.SHA512, expected.SHA512)
 		}
 
 		if err != nil {
-			if task.ignoreMismatch {
+			if ignoreMismatch {
 				downloader.progress.Printf("WARNING: %s\n", err.Error())
 			} else {
 				os.Remove(temppath)
 				return "", err
 			}
+		} else {
+			// update checksums if they match, so that they contain exactly expected set
+			*expected = actual
 		}
 	}
 
 	return temppath, nil
 }
-
-// process implements download thread in goroutine
-func (downloader *downloaderImpl) process() {
-	for {
-		select {
-		case <-downloader.stop:
-			downloader.stopped <- struct{}{}
-			return
-		case <-downloader.pause:
-			<-downloader.unpause
-		case task := <-downloader.queue:
-			downloader.handleTask(task)
-		}
-	}
-}
-
-// DownloadTemp starts new download to temporary file and returns File
-//
-// Temporary file would be already removed, so no need to cleanup
-func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
-	return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false, 1)
-}
-
-// DownloadTempWithChecksum is a DownloadTemp with checksum verification
-//
-// Temporary file would be already removed, so no need to cleanup
-func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
-	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
-	if err != nil {
-		return nil, err
-	}
-	defer os.RemoveAll(tempdir)
-
-	tempfile := filepath.Join(tempdir, "buffer")
-
-	if expected.Size != -1 && downloader.GetProgress() != nil {
-		downloader.GetProgress().InitBar(expected.Size, true)
-		defer downloader.GetProgress().ShutdownBar()
-	}
-
-	ch := make(chan error, 1)
-	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch, maxTries)
-
-	err = <-ch
-
-	if err != nil {
-		return nil, err
-	}
-
-	file, err := os.Open(tempfile)
-	if err != nil {
-		return nil, err
-	}
-
-	return file, nil
-}
-
-// List of extensions + corresponding uncompression support
-var compressionMethods = []struct {
-	extenstion     string
-	transformation func(io.Reader) (io.Reader, error)
-}{
-	{
-		extenstion:     ".bz2",
-		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
-	},
-	{
-		extenstion:     ".gz",
-		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
-	},
-	{
-		extenstion:     ".xz",
-		transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
-	},
-	{
-		extenstion:     "",
-		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
-	},
-}
-
-// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
-// it finds existing file.
-func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
-	var err error
-
-	for _, method := range compressionMethods {
-		var file *os.File
-
-		tryURL := url + method.extenstion
-		foundChecksum := false
-
-		for suffix, expected := range expectedChecksums {
-			if strings.HasSuffix(tryURL, suffix) {
-				file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch, maxTries)
-				foundChecksum = true
-				break
-			}
-		}
-
-		if !foundChecksum {
-			if !ignoreMismatch {
-				continue
-			}
-
-			file, err = DownloadTemp(downloader, tryURL)
-		}
-
-		if err != nil {
-			if err1, ok := err.(*Error); ok && (err1.Code == 404 || err1.Code == 403) {
-				continue
-			}
-			return nil, nil, err
-		}
-
-		var uncompressed io.Reader
-		uncompressed, err = method.transformation(file)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		return uncompressed, file, err
-	}
-
-	if err == nil {
-		err = fmt.Errorf("no candidates for %s found", url)
-	}
-	return nil, nil, err
-}

http/download_test.go

@@ -1,15 +1,12 @@
 package http
 
 import (
-	"errors"
+	"context"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"time"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/console"
@@ -18,17 +15,17 @@ import (
 	. "gopkg.in/check.v1"
 )
 
-type DownloaderSuite struct {
+type DownloaderSuiteBase struct {
 	tempfile *os.File
 	l        net.Listener
 	url      string
-	ch       chan bool
+	ch       chan struct{}
 	progress aptly.Progress
+	d        aptly.Downloader
+	ctx      context.Context
 }
 
-var _ = Suite(&DownloaderSuite{})
+func (s *DownloaderSuiteBase) SetUpTest(c *C) {
-
-func (s *DownloaderSuite) SetUpTest(c *C) {
 	s.tempfile, _ = ioutil.TempFile(os.TempDir(), "aptly-test")
 	s.l, _ = net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
 	s.url = fmt.Sprintf("http://localhost:%d", s.l.Addr().(*net.TCPAddr).Port)
@@ -38,18 +35,21 @@ func (s *DownloaderSuite) SetUpTest(c *C) {
 		fmt.Fprintf(w, "Hello, %s", r.URL.Path)
 	})
 
-	s.ch = make(chan bool)
+	s.ch = make(chan struct{})
 
 	go func() {
 		http.Serve(s.l, mux)
-		s.ch <- true
+		close(s.ch)
 	}()
 
 	s.progress = console.NewProgress()
 	s.progress.Start()
+
+	s.d = NewDownloader(0, s.progress)
+	s.ctx = context.Background()
 }
 
-func (s *DownloaderSuite) TearDownTest(c *C) {
+func (s *DownloaderSuiteBase) TearDownTest(c *C) {
 	s.progress.Shutdown()
 
 	s.l.Close()
@@ -59,249 +59,67 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 	s.tempfile.Close()
 }
 
-func (s *DownloaderSuite) TestStartupShutdown(c *C) {
+type DownloaderSuite struct {
-	goroutines := runtime.NumGoroutine()
+	DownloaderSuiteBase
-
-	d := NewDownloader(10, 100, s.progress)
-	d.Shutdown()
-
-	// wait for goroutines to shutdown
-	time.Sleep(100 * time.Millisecond)
-
-	if runtime.NumGoroutine()-goroutines > 1 {
-		c.Errorf("Number of goroutines %d, expected %d", runtime.NumGoroutine(), goroutines)
-	}
 }
 
-func (s *DownloaderSuite) TestPauseResume(c *C) {
+var _ = Suite(&DownloaderSuite{})
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
 
-	d.Pause()
+func (s *DownloaderSuite) SetUpTest(c *C) {
-	d.Resume()
+	s.DownloaderSuiteBase.SetUpTest(c)
+}
+
+func (s *DownloaderSuite) TearDownTest(c *C) {
+	s.DownloaderSuiteBase.TearDownTest(c)
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", s.tempfile.Name()), IsNil)
-	defer d.Shutdown()
-	ch := make(chan error)
-
-	d.Download(s.url+"/test", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, IsNil)
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false, 1),
-	defer d.Shutdown()
+		ErrorMatches, ".*size check mismatch 12 != 0")
-	ch := make(chan error)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1),
-	res := <-ch
+		ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
-	c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1),
-	res = <-ch
+		IsNil)
-	c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1),
-	res = <-ch
+		IsNil)
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1),
-	res = <-ch
+		ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-	res = <-ch
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1),
-	c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
+		IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1)
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1),
-	res = <-ch
+		ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1)
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
-	res = <-ch
+	c.Assert(s.d.DownloadWithChecksum(s.ctx, s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
-	c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
+		IsNil)
-
+	// download backfills missing checksums
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Check(checksums.SHA512, Equals, "bac18bf4e564856369acc2ed57300fecba3a2c1af5ae8304021e4252488678feb18118466382ee4e1210fe1f065080210e453a80cfb37ccb8752af3269df160e")
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
-	res = <-ch
-	c.Assert(res, IsNil)
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.ctx, s.url+"/doesntexist", s.tempfile.Name()),
-	defer d.Shutdown()
+		ErrorMatches, "HTTP code 404.*")
-	ch := make(chan error)
-
-	d.Download(s.url+"/doesntexist", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, "HTTP code 404.*")
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.ctx, "http://nosuch.localhost/", s.tempfile.Name()),
-	defer d.Shutdown()
+		ErrorMatches, ".*no such host")
-	ch := make(chan error)
-
-	d.Download("http://nosuch.localhost/", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, ".*no such host")
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.ctx, s.url+"/test", "/"),
-	defer d.Shutdown()
+		ErrorMatches, ".*permission denied")
-	ch := make(chan error)
-
-	d.Download(s.url+"/test", "/", ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, ".*permission denied")
-}
-
-func (s *DownloaderSuite) TestDownloadTemp(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTemp(d, s.url+"/test")
-	c.Assert(err, IsNil)
-	defer f.Close()
-
-	buf := make([]byte, 1)
-
-	f.Read(buf)
-	c.Assert(buf, DeepEquals, []byte("H"))
-
-	_, err = os.Stat(f.Name())
-	c.Assert(os.IsNotExist(err), Equals, true)
-}
-
-func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
-	defer f.Close()
-	c.Assert(err, IsNil)
-
-	_, err = DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 13}, false, 1)
-	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
-}
-
-func (s *DownloaderSuite) TestDownloadTempError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTemp(d, s.url+"/doesntexist")
-	c.Assert(err, NotNil)
-	c.Assert(f, IsNil)
-	c.Assert(err, ErrorMatches, "HTTP code 404.*")
-}
-
-const (
-	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
-	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
-	xzData   = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
-	rawData  = "test"
-)
-
-func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
-	var buf []byte
-
-	expectedChecksums := map[string]utils.ChecksumInfo{
-		"file.bz2": {Size: int64(len(bzipData))},
-		"file.gz":  {Size: int64(len(gzipData))},
-		"file.xz":  {Size: int64(len(xzData))},
-		"file":     {Size: int64(len(rawData))},
-	}
-
-	// bzip2 only available
-	buf = make([]byte, 4)
-	d := NewFakeDownloader()
-	d.ExpectResponse("http://example.com/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2 not available, but gz is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.gz", gzipData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2 & gzip not available, but xz is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.xz", xzData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2, gzip & xz not available, but raw is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file", rawData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// gzip available, but broken
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.gz", "x")
-	_, file, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "unexpected EOF")
-	c.Assert(d.Empty(), Equals, true)
-}
-
-func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
-	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "unexpected request.*")
-
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "403")
-
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file", rawData)
-	expectedChecksums := map[string]utils.ChecksumInfo{
-		"file.bz2": {Size: 7},
-		"file.gz":  {Size: 7},
-		"file.xz":  {Size: 7},
-		"file":     {Size: 7},
-	}
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

http/fake.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"context"
 	"fmt"
 	"io"
 	"os"
@@ -60,7 +61,7 @@ func (f *FakeDownloader) Empty() bool {
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
+func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -68,25 +69,21 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 		expectation = f.anyExpected[url]
 		delete(f.anyExpected, url)
 	} else {
-		result <- fmt.Errorf("unexpected request for %s", url)
+		return fmt.Errorf("unexpected request for %s", url)
-		return
 	}
 
 	if expectation.Err != nil {
-		result <- expectation.Err
+		return expectation.Err
-		return
 	}
 
 	err := os.MkdirAll(filepath.Dir(filename), 0755)
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 
 	outfile, err := os.Create(filename)
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 	defer outfile.Close()
 
@@ -95,45 +92,26 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 
 	_, err = w.Write([]byte(expectation.Response))
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 
-	if expected.Size != -1 {
+	if expected != nil {
 		if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
 			expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
 			if ignoreMismatch {
 				fmt.Printf("WARNING: checksums don't match: %#v != %#v for %s\n", expected, cks.Sum(), url)
 			} else {
-				result <- fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
+				return fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
-				return
 			}
 		}
 	}
 
-	result <- nil
+	return nil
-	return
 }
 
 // Download performs fake download by matching against first expectation in the queue
-func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
+func (f *FakeDownloader) Download(ctx context.Context, url string, filename string) error {
-	f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false, 1)
+	return f.DownloadWithChecksum(ctx, url, filename, nil, false, 1)
-}
-
-// Shutdown does nothing
-func (f *FakeDownloader) Shutdown() {
-}
-
-// Abort does nothing
-func (f *FakeDownloader) Abort() {
-}
-
-// Pause does nothing
-func (f *FakeDownloader) Pause() {
-}
-
-// Resume does nothing
-func (f *FakeDownloader) Resume() {
 }
 
 // GetProgress returns Progress object

http/http.go

@@ -1,2 +1,17 @@
 // Package http provides all HTTP (and FTP)-related operations
 package http
+
+import (
+	"fmt"
+)
+
+// Error is download error connected to HTTP code
+type Error struct {
+	Code int
+	URL  string
+}
+
+// Error
+func (e *Error) Error() string {
+	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
+}

http/temp.go

@@ -0,0 +1,48 @@
+package http
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+)
+
+// DownloadTemp starts new download to temporary file and returns File
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTemp(ctx context.Context, downloader aptly.Downloader, url string) (*os.File, error) {
+	return DownloadTempWithChecksum(ctx, downloader, url, nil, false, 1)
+}
+
+// DownloadTempWithChecksum is a DownloadTemp with checksum verification
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTempWithChecksum(ctx context.Context, downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
+	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempdir)
+
+	tempfile := filepath.Join(tempdir, "buffer")
+
+	if expected != nil && downloader.GetProgress() != nil {
+		downloader.GetProgress().InitBar(expected.Size, true)
+		defer downloader.GetProgress().ShutdownBar()
+	}
+
+	err = downloader.DownloadWithChecksum(ctx, url, tempfile, expected, ignoreMismatch, maxTries)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(tempfile)
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}

http/temp_test.go

@@ -0,0 +1,55 @@
+package http
+
+import (
+	"os"
+
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type TempSuite struct {
+	DownloaderSuiteBase
+}
+
+var _ = Suite(&TempSuite{})
+
+func (s *TempSuite) SetUpTest(c *C) {
+	s.DownloaderSuiteBase.SetUpTest(c)
+}
+
+func (s *TempSuite) TearDownTest(c *C) {
+	s.DownloaderSuiteBase.TearDownTest(c)
+}
+
+func (s *TempSuite) TestDownloadTemp(c *C) {
+	f, err := DownloadTemp(s.ctx, s.d, s.url+"/test")
+	c.Assert(err, IsNil)
+	defer f.Close()
+
+	buf := make([]byte, 1)
+
+	f.Read(buf)
+	c.Assert(buf, DeepEquals, []byte("H"))
+
+	_, err = os.Stat(f.Name())
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *TempSuite) TestDownloadTempWithChecksum(c *C) {
+	f, err := DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
+	c.Assert(err, IsNil)
+
+	c.Assert(f.Close(), IsNil)
+
+	_, err = DownloadTempWithChecksum(s.ctx, s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false, 1)
+	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
+}
+
+func (s *TempSuite) TestDownloadTempError(c *C) {
+	f, err := DownloadTemp(s.ctx, s.d, s.url+"/doesntexist")
+	c.Assert(err, NotNil)
+	c.Assert(f, IsNil)
+	c.Assert(err, ErrorMatches, "HTTP code 404.*")
+}

linter.json

@@ -1,4 +1,24 @@
 {
     "DisableAll": true,
-    "Enable": ["vet", "golint", "gofmt", "deadcode", "goimports", "misspell"]
+    "Enable": [
+        "vet",
+        "golint",
+        "gofmt",
+        "deadcode",
+        "goimports",
+        "misspell",
+        "gosimple",
+        "ineffassign",
+        "staticcheck",
+        "varcheck",
+        "structcheck",
+        "maligned",
+        "vetshadow",
+        "goconst",
+        "interfacer"
+    ],
+    "Deadline": "20m",
+    "Vendor": true,
+    "VendoredLinters": true,
+    "Concurrency": 1
 }

lock.json

@@ -1,292 +0,0 @@
-{
-    "memo": "4ce49ad4105227467bccc48eec5832d84749da3dee8dc8f6574cd32fec3970c1",
-    "projects": [
-        {
-            "name": "github.com/AlekSi/pointer",
-            "version": "v1.0.0",
-            "revision": "08a25bac605b3fcb6cc27f3917b2c2c87451963d",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/DisposaBoy/JsonConfigReader",
-            "branch": "master",
-            "revision": "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/awalterschulze/gographviz",
-            "version": "v1.0",
-            "revision": "761fd5fbb34e4c2c138c280395b65b48e4ff5a53",
-            "packages": [
-                ".",
-                "ast",
-                "parser",
-                "scanner",
-                "token"
-            ]
-        },
-        {
-            "name": "github.com/aws/aws-sdk-go",
-            "version": "v1.8.0",
-            "revision": "2db5849d2939d93075d911138309a83235032bea",
-            "packages": [
-                "aws",
-                "aws/awserr",
-                "aws/awsutil",
-                "aws/client",
-                "aws/client/metadata",
-                "aws/corehandlers",
-                "aws/credentials",
-                "aws/credentials/ec2rolecreds",
-                "aws/credentials/endpointcreds",
-                "aws/credentials/stscreds",
-                "aws/defaults",
-                "aws/ec2metadata",
-                "aws/endpoints",
-                "aws/request",
-                "aws/session",
-                "aws/signer/v4",
-                "private/protocol",
-                "private/protocol/query",
-                "private/protocol/query/queryutil",
-                "private/protocol/rest",
-                "private/protocol/restxml",
-                "private/protocol/xml/xmlutil",
-                "service/s3",
-                "service/sts"
-            ]
-        },
-        {
-            "name": "github.com/cheggaaa/pb",
-            "version": "v1.0.10",
-            "revision": "cdf719fac0dd208251aa828e687c2d5802053b51",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/gin-gonic/gin",
-            "revision": "b1758d3bfa09e61ddbc1c9a627e936eec6a170de",
-            "packages": [
-                ".",
-                "binding",
-                "render"
-            ]
-        },
-        {
-            "name": "github.com/go-ini/ini",
-            "version": "v1.26.0",
-            "revision": "1730955e3146956d6a087861380f9b4667ed5071",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/golang/snappy",
-            "branch": "master",
-            "revision": "553a641470496b2327abcac10b36396bd98e45c9",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/h2non/filetype",
-            "branch": "master",
-            "revision": "0df83c38d14ff5f653d419d480eaac286ccbc823",
-            "packages": [
-                "matchers"
-            ]
-        },
-        {
-            "name": "github.com/jlaffaye/ftp",
-            "branch": "master",
-            "revision": "7b85eb4638a2c0473acefcfb929a98f879c15c86",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/jmespath/go-jmespath",
-            "version": "0.2.2",
-            "revision": "3433f3ea46d9f8019119e7dd41274e112a2359a9",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/julienschmidt/httprouter",
-            "version": "v1.1",
-            "revision": "8c199fb6259ffc1af525cc3ad52ee60ba8359669",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mattn/go-runewidth",
-            "version": "v0.0.2",
-            "revision": "9e777a8366cce605130a531d2cd6363d07ad7317",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mattn/go-shellwords",
-            "version": "v1.0.2",
-            "revision": "005a0944d84452842197c2108bd9168ced206f78",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mkrautz/goar",
-            "branch": "master",
-            "revision": "282caa8bd9daba480b51f1d5a988714913b97aad",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mxk/go-flowrate",
-            "branch": "master",
-            "revision": "cca7078d478f8520f85629ad7c68962d31ed7682",
-            "packages": [
-                "flowrate"
-            ]
-        },
-        {
-            "name": "github.com/ncw/swift",
-            "branch": "master",
-            "revision": "8e9b10220613abdbc2896808ee6b43e411a4fa6c",
-            "packages": [
-                ".",
-                "swifttest"
-            ]
-        },
-        {
-            "name": "github.com/smira/commander",
-            "branch": "master",
-            "revision": "f408b00e68d5d6e21b9f18bd310978dafc604e47",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/flag",
-            "branch": "master",
-            "revision": "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/go-aws-auth",
-            "branch": "master",
-            "revision": "0070896e9d7f4f9f2d558532b2d896ce2239992a",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/go-ftp-protocol",
-            "branch": "master",
-            "revision": "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa",
-            "packages": [
-                "protocol"
-            ]
-        },
-        {
-            "name": "github.com/smira/go-uuid",
-            "branch": "master",
-            "revision": "ed3ca8a15a931b141440a7e98e4f716eec255f7d",
-            "packages": [
-                "uuid"
-            ]
-        },
-        {
-            "name": "github.com/smira/go-xz",
-            "branch": "master",
-            "revision": "0c531f070014e218b21f3cfca801cc992d52726d",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/lzma",
-            "branch": "master",
-            "revision": "7f0af6269940baa2c938fabe73e0d7ba41205683",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/syndtr/goleveldb",
-            "branch": "master",
-            "revision": "3c5717caf1475fd25964109a0fc640bd150fce43",
-            "packages": [
-                "leveldb",
-                "leveldb/cache",
-                "leveldb/comparer",
-                "leveldb/errors",
-                "leveldb/filter",
-                "leveldb/iterator",
-                "leveldb/journal",
-                "leveldb/memdb",
-                "leveldb/opt",
-                "leveldb/storage",
-                "leveldb/table",
-                "leveldb/util"
-            ]
-        },
-        {
-            "name": "github.com/ugorji/go",
-            "revision": "71c2886f5a673a35f909803f38ece5810165097b",
-            "packages": [
-                "codec"
-            ]
-        },
-        {
-            "name": "github.com/wsxiaoys/terminal",
-            "branch": "master",
-            "revision": "0940f3fc43a0ed42d04916b1c04578462c650b09",
-            "packages": [
-                "color"
-            ]
-        },
-        {
-            "name": "golang.org/x/crypto",
-            "branch": "master",
-            "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7",
-            "packages": [
-                "ssh/terminal"
-            ]
-        },
-        {
-            "name": "golang.org/x/sys",
-            "branch": "master",
-            "revision": "99f16d856c9836c42d24e7ab64ea72916925fa97",
-            "packages": [
-                "unix"
-            ]
-        },
-        {
-            "name": "gopkg.in/check.v1",
-            "branch": "v1",
-            "revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "gopkg.in/h2non/filetype.v1",
-            "version": "v1.0.1",
-            "revision": "3093b8ebec6efb56ac813238b8beab4ed4eaac6a",
-            "packages": [
-                "types"
-            ]
-        }
-    ]
-}