Merge pull request #614 from smira/600-fix-double-mirror-update

Fix bug with `PoolPath` field being overwritten on mirror update

.gitignore

@@ -34,3 +34,5 @@ man/aptly.1.html
 man/aptly.1.ronn
 
 .goxc.local.json
+
+system/env/

.goxc.json

@@ -10,17 +10,17 @@
 		"bintray"
     ],
     "TaskSettings": {
-        "deb": {
+        "debs": {
             "metadata": {
                 "maintainer": "Andrey Smirnov",
-                "maintainerEmail": "me@smira.ru",
+                "maintainer-email": "me@smira.ru",
                 "description": "Debian repository management tool"
             },
             "metadata-deb": {
                 "License": "MIT",
                 "Homepage": "https://www.aptly.info/",
-                "Recommends": "bzip2, graphviz, xz-utils",
+                "Depends": "bzip2, xz-utils, gnupg, gpgv",
-                "Depends": ""
+				"Suggests": "graphviz"
             },
             "other-mapped-files": {
                 "/": "root/"

.travis.yml

@@ -1,12 +1,15 @@
-sudo: false
+dist: trusty
+sudo: required
 
 language: go
 
 go:
-  - 1.6
+  - 1.6.x
-  - 1.7
+  - 1.7.x
-  - 1.8
+  - 1.8.x
-  - tip
+  - master
+
+go_import_path: github.com/smira/aptly
 
 addons:
   apt:
@@ -20,14 +23,11 @@ env:
     - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
     - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
 before_install:
-  - virtualenv env
+  - virtualenv system/env
-  - . env/bin/activate
+  - . system/env/bin/activate
   - pip install six packaging appdirs
   - pip install -U pip setuptools
-  - pip install boto requests requests-unixsocket python-swiftclient
+  - pip install -r system/requirements.txt
-  - mkdir -p $GOPATH/src/github.com/smira
-  - ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/smira || true
-  - cd $GOPATH/src/github.com/smira/aptly
   - make version
 install:
   - make prepare
@@ -36,7 +36,7 @@ script: make travis
 
 matrix:
     allow_failures:
-        - go: tip
+        - go: master
 
 notifications:
   webhooks:

AUTHORS

@@ -26,3 +26,5 @@ List of contributors, in chronological order:
 * Harald Sitter (https://github.com/apachelogger)
 * Johannes Layher (https://github.com/jola5)
 * Charles Hsu (https://github.com/charz)
+* Clemens Rabe (https://github.com/seeraven)
+* TJ Merritt (https://github.com/tjmerritt)

CONTRIBUTING.md

@@ -0,0 +1,239 @@
+# Contributing to aptly
+
+:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
+
+The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
+These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
+
+## What should I know before I get started?
+
+### Code of Conduct
+
+This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to uphold this code.
+Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
+
+### List of Repositories
+
+* [smira/aptly](https://github.com/smira/aptly) - aptly source code, functional tests, man page
+* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
+  fixtures for aptly functional tests
+
+## How Can I Contribute?
+
+### Reporting Bugs
+
+1. Please search for similar bug report in [issue tracker](https://github.com/smira/aptly/issues)
+2. Please verify that bug is not fixed in latest aptly nightly ([download information](https://www.aptly.info/download/))
+3. Steps to reproduce increases chances for bug to be fixed quickly. If possible, submit PR with new functional test which fails.
+4. If bug is reproducible with specific package, please provide link to package file.
+5. Open issue at [GitHub](https://github.com/smira/aptly/issues)
+
+### Suggesting Enhancements
+
+1. Please search [issue tracker](https://github.com/smira/aptly/issues) for similar feature requests.
+2. Describe why enhancement is important to you.
+3. Include any additional details or implementation details.
+
+### Improving Documentation
+
+There are two kinds of documentation:
+
+* [aptly website](https://www.aptly/info)
+* aptly `man` page
+
+Core content is mostly the same, but website contains more information, tutorials, examples.
+
+If you want to update `man` page, please open PR to [main aptly repo](https://github.com/smira/aptly),
+details in [man page](#man-page) section.
+
+If you want to update website, please follow steps below:
+
+1. Install [hugo](http://gohugo.io/)
+2. Fork [website source](https://github.com/aptly-dev/aptly-dev.github.io) and clone it
+3. Launch hugo in development mode: `hugo -w server`
+4. Navigate to `http://localhost:1313/`: you should see aptly website
+5. Update documentation, most of the time editing Markdown is all you need.
+6. Page in browser should reload automatically as you make changes to source files.
+
+We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
+general fixes, clarifications, misspellings, grammar mistakes!
+
+### Your Fist Code Contribution
+
+Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
+following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
+
+Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
+if that is possible.
+
+## Development Setup
+
+This section describes local setup to start contributing to aptly source.
+
+### Go & Python
+
+You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
+
+If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
+initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
+
+Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
+to prepend it or to skip this test if you're security conscious.  
+
+### Forking and Cloning
+
+As Go is using repository path in import paths, it's better to clone aptly repo (not your fork) at default location:
+
+    mkdir -p ~/go/src/github.com/smira
+    cd ~/go/src/github.com/smira
+    git clone git@github.com:smira/aptly.git
+    cd aptly
+
+For main repo under your GitHub user and add it as another Git remote:
+
+    git remote add <user> git@github.com:<user>/aptly.git
+
+That way you can continue to build project as is (you don't need to adjust import paths), but you would need
+to specify your remote name when pushing branches:
+
+    git push <user> <your-branch>
+
+### Dependencies
+
+You would need some additional tools and Python virtual environment to run tests and checks, install them with:
+
+    make prepare dev system/env
+
+This is usually one-time action.
+
+### Building
+
+If you want to build aptly binary from your current source tree, run:
+
+    make install
+
+This would build `aptly` in `$GOPATH/bin`, so depending on your `$PATH`, you should be able to run it immediately with:
+
+    aptly
+
+Or, if it's not on your path:
+
+    ~/go/bin/aptly
+
+### Unit-tests
+
+aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
+feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
+
+aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.org/gocheck). Run the unit-tests with:
+
+    make test
+
+### Functional Tests
+
+Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
+runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
+run some aptly commands capturing output, exit code, checking any additional files being created and so on. API tests
+are a bit different, as they re-use same aptly process serving API requests.
+
+The easiest way to run functional tests is to use `make`:
+
+    make system-test
+
+This would check all the dependencies and run all the tests. Some tests (S3, Swift) require access credentials to
+be set up in the environment. For example, it needs AWS credentials to run S3 tests (they would be used to publish to S3).
+If credentials are missing, tests would be skipped.
+
+You can also run subset of tests manually:
+
+    system/run.py t04_mirror
+
+This would run all the mirroring tests under `system/t04_mirror` folder.
+
+Or you can run tests by test name mask:
+
+    system/run.py UpdateMirror*
+
+Or, you can run specific test by name:
+
+    system/run.py UpdateMirror7Test
+
+Test runner can update expected output instead of failing on mismatch (this is especially useful while
+working on new tests):
+
+    system/run.py --capture <test>
+
+Output for some tests might contain environment-specific things, e.g. your home directory. In that case
+you can use `${HOME}` and similar variable expansion in expected output files.
+
+Some tests depend on fixtures, for example pre-populated GPG trusted keys. There are also test fixtures
+captured after mirror update which contain pre-build aptly database and pool contents. They're useful if you
+don't want to waste time in the test on populating aptly database while you need some packages to work with.
+There are some packages available under `system/files/` directory which are used to build contents of local repos.
+
+*WARNING*: tests are running under current `$HOME` directory with aptly default settings, so they clear completely
+`~/.aptly.conf` and `~/.aptly` subdirectory between the runs. So it's not wise to have non-dev aptly being used with
+this default location. You can run aptly under different user or by using non-default config location with non-default
+aptly root directory.
+
+### Style Checks
+
+Style checks could be run with:
+
+    make check
+
+aptly is using [gometalinter](https://github.com/alecthomas/gometalinter) to run style checks on Go code. Configuration
+for the linter could be found in [linter.json](linter.json) file. Running linters might take considerable amount of time
+unfortunately, but usually warning reported by linters hint at real code issues.
+
+Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
+
+### Vendored Code
+
+aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
+repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
+packages over packages in `$GOPATH`.
+
+If you want to update vendored dependencies or to introduce new dependency, use [dep tool](https://github.com/golang/dep).
+Usually all you need is `dep ensure` or `dep ensure -update`.
+
+### man Page
+
+aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source
+template [man/aptly.1.ronn.tmpl](man/aptly.1.ronn.tmpl) is changed or any command help is changed, run `make man` to regenerate
+final rendered man page [man/aptly.1](man/aptly.1). In the end of the build, new man page is displayed for visual
+verification.
+
+Man page is built with small helper [_man/gen.go](man/gen.go) which pulls in template, command-line help from [cmd/](cmd/) folder
+and runs that through [forked copy](https://github.com/smira/ronn) of [ronn](https://github.com/rtomayko/ronn).
+
+### Bash Completion
+
+Bash completion for aptly resides in the same repo under in [bash_completion.d/aptly](bash_completion.d/aptly). It's all hand-crafted.
+When new option or command is introduced, bash completion should be updated to reflect that change.
+
+When aptly package is being built, it automatically pulls bash completion and man page into the package.
+
+## Design
+
+This section requires future work.
+
+*TBD*
+
+### Database
+
+### Package Pool
+
+### Package
+
+### PackageList, PackageRefList
+
+### LocalRepo, RemoteRepo, Snapshot
+
+### PublishedRepository
+
+### Context
+
+### Collections, CollectionFactory

Gopkg.lock

@@ -0,0 +1,191 @@
+memo = "becdf010a814559719c990c1bd645c737cee332ad52004c440605c13de100d45"
+
+[[projects]]
+  name = "github.com/AlekSi/pointer"
+  packages = ["."]
+  revision = "08a25bac605b3fcb6cc27f3917b2c2c87451963d"
+  version = "v1.0.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/DisposaBoy/JsonConfigReader"
+  packages = ["."]
+  revision = "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4"
+
+[[projects]]
+  name = "github.com/awalterschulze/gographviz"
+  packages = [".","ast","parser","scanner","token"]
+  revision = "761fd5fbb34e4c2c138c280395b65b48e4ff5a53"
+  version = "v1.0"
+
+[[projects]]
+  name = "github.com/aws/aws-sdk-go"
+  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
+  revision = "2db5849d2939d93075d911138309a83235032bea"
+  version = "v1.8.0"
+
+[[projects]]
+  name = "github.com/cheggaaa/pb"
+  packages = ["."]
+  revision = "cdf719fac0dd208251aa828e687c2d5802053b51"
+  version = "v1.0.10"
+
+[[projects]]
+  name = "github.com/gin-gonic/gin"
+  packages = [".","binding","render"]
+  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
+
+[[projects]]
+  name = "github.com/go-ini/ini"
+  packages = ["."]
+  revision = "1730955e3146956d6a087861380f9b4667ed5071"
+  version = "v1.26.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/golang/snappy"
+  packages = ["."]
+  revision = "553a641470496b2327abcac10b36396bd98e45c9"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/h2non/filetype"
+  packages = ["matchers"]
+  revision = "0df83c38d14ff5f653d419d480eaac286ccbc823"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/jlaffaye/ftp"
+  packages = ["."]
+  revision = "7b85eb4638a2c0473acefcfb929a98f879c15c86"
+
+[[projects]]
+  name = "github.com/jmespath/go-jmespath"
+  packages = ["."]
+  revision = "3433f3ea46d9f8019119e7dd41274e112a2359a9"
+  version = "0.2.2"
+
+[[projects]]
+  name = "github.com/julienschmidt/httprouter"
+  packages = ["."]
+  revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
+  version = "v1.1"
+
+[[projects]]
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  revision = "9e777a8366cce605130a531d2cd6363d07ad7317"
+  version = "v0.0.2"
+
+[[projects]]
+  name = "github.com/mattn/go-shellwords"
+  packages = ["."]
+  revision = "005a0944d84452842197c2108bd9168ced206f78"
+  version = "v1.0.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mkrautz/goar"
+  packages = ["."]
+  revision = "282caa8bd9daba480b51f1d5a988714913b97aad"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/mxk/go-flowrate"
+  packages = ["flowrate"]
+  revision = "cca7078d478f8520f85629ad7c68962d31ed7682"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/ncw/swift"
+  packages = [".","swifttest"]
+  revision = "8e9b10220613abdbc2896808ee6b43e411a4fa6c"
+
+[[projects]]
+  name = "github.com/pkg/errors"
+  packages = ["."]
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/commander"
+  packages = ["."]
+  revision = "f408b00e68d5d6e21b9f18bd310978dafc604e47"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/flag"
+  packages = ["."]
+  revision = "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-aws-auth"
+  packages = ["."]
+  revision = "0070896e9d7f4f9f2d558532b2d896ce2239992a"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-ftp-protocol"
+  packages = ["protocol"]
+  revision = "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-uuid"
+  packages = ["uuid"]
+  revision = "ed3ca8a15a931b141440a7e98e4f716eec255f7d"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/go-xz"
+  packages = ["."]
+  revision = "0c531f070014e218b21f3cfca801cc992d52726d"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/smira/lzma"
+  packages = ["."]
+  revision = "7f0af6269940baa2c938fabe73e0d7ba41205683"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/syndtr/goleveldb"
+  packages = ["leveldb","leveldb/cache","leveldb/comparer","leveldb/errors","leveldb/filter","leveldb/iterator","leveldb/journal","leveldb/memdb","leveldb/opt","leveldb/storage","leveldb/table","leveldb/util"]
+  revision = "3c5717caf1475fd25964109a0fc640bd150fce43"
+
+[[projects]]
+  name = "github.com/ugorji/go"
+  packages = ["codec"]
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/wsxiaoys/terminal"
+  packages = ["color"]
+  revision = "0940f3fc43a0ed42d04916b1c04578462c650b09"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+  packages = ["cast5","openpgp","openpgp/armor","openpgp/clearsign","openpgp/elgamal","openpgp/errors","openpgp/packet","openpgp/s2k","ssh/terminal"]
+  revision = "459e26527287adbc2adcc5d0d49abff9a5f315a7"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/sys"
+  packages = ["unix"]
+  revision = "99f16d856c9836c42d24e7ab64ea72916925fa97"
+
+[[projects]]
+  branch = "v1"
+  name = "gopkg.in/check.v1"
+  packages = ["."]
+  revision = "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec"
+
+[[projects]]
+  name = "gopkg.in/h2non/filetype.v1"
+  packages = ["types"]
+  revision = "3093b8ebec6efb56ac813238b8beab4ed4eaac6a"
+  version = "v1.0.1"

Gopkg.toml

@@ -0,0 +1,32 @@
+
+[[dependencies]]
+  name = "github.com/gin-gonic/gin"
+  revision = "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/mkrautz/goar"
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/smira/go-uuid"
+
+[[dependencies]]
+  branch = "master"
+  name = "github.com/smira/go-xz"
+
+[[dependencies]]
+  name = "github.com/ugorji/go"
+  revision = "71c2886f5a673a35f909803f38ece5810165097b"
+
+[[dependencies]]
+  branch = "master"
+  name = "golang.org/x/crypto"
+
+[[dependencies]]
+  branch = "master"
+  name = "golang.org/x/sys"
+
+[[dependencies]]
+  branch = "v1"
+  name = "gopkg.in/check.v1"

Makefile

@@ -1,6 +1,6 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
 VERSION=$(shell git describe --tags | sed 's@^v@@' | sed 's@-@+@g')
-PACKAGES=context database deb files http query swift s3 utils
+PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python
 TESTS?=
 BINPATH?=$(GOPATH)/bin
@@ -35,16 +35,26 @@ coverage: coverage.out
 	go tool cover -html=coverage.out
 	rm -f coverage.out
 
-check:
+check: system/env
-	gometalinter --vendor --vendored-linters --config=linter.json ./...
+	if [ -x travis_wait ]; then \
+		travis_wait gometalinter --config=linter.json ./...; \
+	else \
+		gometalinter --config=linter.json ./...; \
+	fi
+	. system/env/bin/activate && flake8 --max-line-length=200 --exclude=system/env/ system/
 
 install:
 	go install -v -ldflags "-X main.Version=$(VERSION)"
 
-system-test: install
+system/env: system/requirements.txt
+	rm -rf system/env
+	virtualenv system/env
+	system/env/bin/pip install -r system/requirements.txt
+
+system-test: install system/env
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
+	. system/env/bin/activate && APTLY_VERSION=$(VERSION) PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long $(TESTS)
 
 travis: $(TRAVIS_TARGET) check system-test
 
@@ -58,20 +68,11 @@ mem.png: mem.dat mem.gp
 	gnuplot mem.gp
 	open mem.png
 
-src-package:
-	rm -rf aptly-$(VERSION)
-	mkdir -p aptly-$(VERSION)/src/github.com/smira/aptly/
-	cd aptly-$(VERSION)/src/github.com/smira/ && git clone https://github.com/smira/aptly && cd aptly && git checkout v$(VERSION)
-	mkdir -p aptly-$(VERSION)/bash_completion.d
-	(cd aptly-$(VERSION)/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/$(VERSION)/aptly)
-	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
-	rm -rf aptly-$(VERSION)
-
 goxc:
 	rm -rf root/
 	mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d
 	cp man/aptly.1 root/usr/share/man/man1
-	(cd root/etc/bash_completion.d && wget https://raw.github.com/aptly-dev/aptly-bash-completion/master/aptly)
+	cp bash_completion.d/aptly root/etc/bash_completion.d
 	gzip root/usr/share/man/man1/aptly.1
 	goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS)
 

README.rst

@@ -2,11 +2,11 @@
 aptly
 =====
 
-.. image:: https://travis-ci.org/smira/aptly.png?branch=master
+.. image:: https://api.travis-ci.org/smira/aptly.svg?branch=master
     :target: https://travis-ci.org/smira/aptly
 
-.. image:: https://coveralls.io/repos/smira/aptly/badge.png?branch=HEAD
+.. image:: https://coveralls.io/repos/smira/aptly/badge.svg?branch=master
-    :target: https://coveralls.io/r/smira/aptly?branch=HEAD
+    :target: https://coveralls.io/r/smira/aptly?branch=master
 
 .. image:: https://badges.gitter.im/Join Chat.svg
     :target: https://gitter.im/smira/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
@@ -19,7 +19,7 @@ Aptly is a swiss army knife for Debian repository management.
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
-Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
+Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
 Aptly features: ("+" means planned features)
@@ -42,7 +42,7 @@ Current limitations:
 Download
 --------
 
-To install aptly on Debian/Ubuntu, add new repository to /etc/apt/sources.list::
+To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
 
     deb http://repo.aptly.info/ squeeze main
 
@@ -73,6 +73,11 @@ If you have Go environment set up, you can build aptly from source by running (g
 
 Binary would be installed to ```$GOPATH/bin/aptly``.
 
+Contributing
+------------
+
+Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+
 Integrations
 ------------
 

api/api.go

@@ -23,11 +23,18 @@ func apiVersion(c *gin.Context) {
 	c.JSON(200, gin.H{"Version": aptly.Version})
 }
 
+type dbRequestKind int
+
 const (
-	acquiredb = iota
+	acquiredb dbRequestKind = iota
 	releasedb
 )
 
+type dbRequest struct {
+	kind dbRequestKind
+	err  chan<- error
+}
+
 // Flushes all collections which cache in-memory objects
 func flushColections() {
 	// lock everything to eliminate in-progress calls
@@ -52,50 +59,48 @@ func flushColections() {
 }
 
 // Periodically flushes CollectionFactory to free up memory used by
-// collections, flushing caches. If the two channels are provided,
+// collections, flushing caches.
-// they are used to acquire and release the database.
 //
 // Should be run in goroutine!
-func cacheFlusher(requests chan int, acks chan error) {
+func cacheFlusher() {
 	ticker := time.Tick(15 * time.Minute)
 
 	for {
 		<-ticker
 
-		// if aptly API runs in -no-lock mode,
-		// caches are flushed when DB is closed anyway, no need
-		// to flush them here
-		if requests == nil {
 		flushColections()
 	}
 }
-}
 
-// Acquire database lock and release it when not needed anymore. Two
+// Acquire database lock and release it when not needed anymore.
-// channels must be provided. The first one is to receive requests to
-// acquire/release the database and the second one is to send acks.
 //
 // Should be run in a goroutine!
-func acquireDatabase(requests chan int, acks chan error) {
+func acquireDatabase(requests <-chan dbRequest) {
 	clients := 0
-	for {
+	for request := range requests {
-		request := <-requests
+		var err error
-		switch request {
+
+		switch request.kind {
 		case acquiredb:
 			if clients == 0 {
-				acks <- context.ReOpenDatabase()
+				err = context.ReOpenDatabase()
-			} else {
-				acks <- nil
 			}
+
+			request.err <- err
+
+			if err == nil {
 				clients++
+			}
 		case releasedb:
 			clients--
 			if clients == 0 {
 				flushColections()
-				acks <- context.CloseDatabase()
+				err = context.CloseDatabase()
 			} else {
-				acks <- nil
+				err = nil
 			}
+
+			request.err <- err
 		}
 	}
 }

api/publish.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -20,12 +21,12 @@ type SigningOptions struct {
 	PassphraseFile string
 }
 
-func getSigner(options *SigningOptions) (utils.Signer, error) {
+func getSigner(options *SigningOptions) (pgp.Signer, error) {
 	if options.Skip {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(options.GpgKey)
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
@@ -97,6 +98,8 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		Distribution         string
 		Label                string
 		Origin               string
+		NotAutomatic         string
+		ButAutomaticUpgrades string
 		ForceOverwrite       bool
 		SkipContents         *bool
 		Architectures        []string
@@ -145,7 +148,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 
 			sources = append(sources, snapshot)
 		}
-	} else if b.SourceKind == "local" {
+	} else if b.SourceKind == deb.SourceLocalRepo {
 		var localRepo *deb.LocalRepo
 
 		localCollection := context.CollectionFactory().LocalRepoCollection()
@@ -182,7 +185,15 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		c.Fail(500, fmt.Errorf("unable to publish: %s", err))
 		return
 	}
+	if b.Origin != "" {
 		published.Origin = b.Origin
+	}
+	if b.NotAutomatic != "" {
+		published.NotAutomatic = b.NotAutomatic
+	}
+	if b.ButAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
+	}
 	published.Label = b.Label
 
 	published.SkipContents = context.Config().SkipContentsPublishing
@@ -264,7 +275,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 	var updatedComponents []string
 
-	if published.SourceKind == "local" {
+	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			c.Fail(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
@@ -281,15 +292,15 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 				return
 			}
 
-			snapshot, err := snapshotCollection.ByName(snapshotInfo.Name)
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
 			if err != nil {
-				c.Fail(404, err)
+				c.Fail(404, err2)
 				return
 			}
 
-			err = snapshotCollection.LoadComplete(snapshot)
+			err2 = snapshotCollection.LoadComplete(snapshot)
-			if err != nil {
+			if err2 != nil {
-				c.Fail(500, err)
+				c.Fail(500, err2)
 				return
 			}
 

api/repos.go

@@ -296,7 +296,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
 	var (
 		sources                      []string
@@ -325,7 +325,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	}
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), reporter, nil)
+		context.CollectionFactory().PackageCollection(), reporter, nil, context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 
 	if err != nil {

api/router.go

@@ -20,32 +20,35 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
 		// running, we close the database to free the lock.
-		requests := make(chan int)
+		requests := make(chan dbRequest)
-		acks := make(chan error)
 
-		go acquireDatabase(requests, acks)
+		go acquireDatabase(requests)
-		go cacheFlusher(requests, acks)
 
 		router.Use(func(c *gin.Context) {
-			requests <- acquiredb
+			var err error
-			err := <-acks
+
+			errCh := make(chan error)
+			requests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
 			if err != nil {
 				c.Fail(500, err)
 				return
 			}
+
 			defer func() {
-				requests <- releasedb
+				requests <- dbRequest{releasedb, errCh}
-				err = <-acks
+				err = <-errCh
 				if err != nil {
 					c.Fail(500, err)
-					return
 				}
 			}()
+
 			c.Next()
 		})
 
 	} else {
-		go cacheFlusher(nil, nil)
+		go cacheFlusher()
 	}
 
 	root := router.Group("/api")

aptly/interfaces.go

@@ -4,24 +4,57 @@ package aptly
 
 import (
 	"io"
+	"os"
 
 	"github.com/smira/aptly/utils"
 )
 
+// ReadSeekerCloser = ReadSeeker + Closer
+type ReadSeekerCloser interface {
+	io.ReadSeeker
+	io.Closer
+}
+
 // PackagePool is asbtraction of package pool storage.
 //
 // PackagePool stores all the package files, deduplicating them.
 type PackagePool interface {
-	// Path returns full path to package file in pool given any name and hash of file contents
+	// Verify checks whether file exists in the pool and fills back checksum info
-	Path(filename string, hashMD5 string) (string, error)
+	//
-	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+	// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
-	RelativePath(filename string, hashMD5 string) (string, error)
+	// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+	Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage ChecksumStorage) (string, bool, error)
+	// Import copies file into package pool
+	//
+	// - srcPath is full path to source file as it is now
+	// - basename is desired human-readable name (canonical filename)
+	// - checksums are used to calculate file placement
+	// - move indicates whether srcPath can be removed
+	Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
+	// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
+	LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
+	// Stat returns Unix stat(2) info
+	Stat(path string) (os.FileInfo, error)
+	// Open returns ReadSeekerCloser to access the file
+	Open(path string) (ReadSeekerCloser, error)
 	// FilepathList returns file paths of all the files in the pool
 	FilepathList(progress Progress) ([]string, error)
 	// Remove deletes file in package pool returns its size
 	Remove(path string) (size int64, err error)
-	// Import copies file into package pool
+}
-	Import(path string, hashMD5 string) error
+
+// LocalPackagePool is implemented by PackagePools residing on the same filesystem
+type LocalPackagePool interface {
+	// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+	GenerateTempPath(filename string) (string, error)
+	// Link generates hardlink to destination path
+	Link(path, dstPath string) error
+	// Symlink generates symlink to destination path
+	Symlink(path, dstPath string) error
+	// FullPath generates full path to the file in pool
+	//
+	// Please use with care: it's not supposed to be used to access files
+	FullPath(path string) string
 }
 
 // PublishedStorage is abstraction of filesystem storing all published repositories
@@ -35,15 +68,15 @@ type PublishedStorage interface {
 	// Remove removes single file under public path
 	Remove(path string) error
 	// LinkFromPool links package file from pool to dist's pool location
-	LinkFromPool(publishedDirectory string, sourcePool PackagePool, sourcePath, sourceMD5 string, force bool) error
+	LinkFromPool(publishedDirectory, baseName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
 	// Filelist returns list of files under prefix
 	Filelist(prefix string) ([]string, error)
 	// RenameFile renames (moves) file
 	RenameFile(oldName, newName string) error
 }
 
-// LocalPublishedStorage is published storage on local filesystem
+// FileSystemPublishedStorage is published storage on filesystem
-type LocalPublishedStorage interface {
+type FileSystemPublishedStorage interface {
 	// PublicPath returns root of public part
 	PublicPath() string
 }
@@ -76,23 +109,24 @@ type Progress interface {
 	Printf(msg string, a ...interface{})
 	// ColoredPrintf does printf in colored way + newline
 	ColoredPrintf(msg string, a ...interface{})
+	// PrintfStdErr does printf but in safe manner to stderr
+	PrintfStdErr(msg string, a ...interface{})
 }
 
 // Downloader is parallel HTTP fetcher
 type Downloader interface {
 	// Download starts new download task
-	Download(url string, destination string, result chan<- error)
+	Download(url string, destination string) error
 	// DownloadWithChecksum starts new download task with checksum verification
-	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int)
+	DownloadWithChecksum(url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error
-	// Pause pauses task processing
-	Pause()
-	// Resume resumes task processing
-	Resume()
-	// Shutdown stops downloader after current tasks are finished,
-	// but doesn't process rest of queue
-	Shutdown()
-	// Abort stops downloader without waiting for shutdown
-	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }
+
+// ChecksumStorage is stores checksums in some (persistent) storage
+type ChecksumStorage interface {
+	// Get finds checksums in DB by path
+	Get(path string) (*utils.ChecksumInfo, error)
+	// Update adds or updates information about checksum in DB
+	Update(path string, c *utils.ChecksumInfo) error
+}

bash_completion.d/aptly

@@ -0,0 +1,633 @@
+#!/bin/bash
+
+# (The MIT License)
+#
+# Copyright (c) 2014 Andrey Smirnov
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+__aptly_mirror_list()
+{
+  aptly mirror list -raw
+}
+
+__aptly_repo_list()
+{
+  aptly repo list -raw
+}
+
+__aptly_snapshot_list()
+{
+  aptly snapshot list -raw
+}
+
+__aptly_published_distributions()
+{
+  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
+}
+
+__aptly_published_prefixes()
+{
+  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
+}
+
+__aptly_prefixes_for_distribution()
+{
+  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
+}
+
+_aptly()
+{
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+
+    commands="api config db graph mirror package publish repo serve snapshot task version"
+    options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
+    db_subcommands="cleanup recover"
+    mirror_subcommands="create drop edit show list rename search update"
+    publish_subcommands="drop list repo snapshot switch update"
+    snapshot_subcommands="create diff drop filter list merge pull rename search show verify"
+    repo_subcommands="add copy create drop edit import include list move remove rename search show"
+    package_subcommands="search show"
+    task_subcommands="run"
+    config_subcommands="show"
+    api_subcommands="serve"
+
+    local cmd subcmd numargs numoptions i
+
+    numargs=0
+    numoptions=0
+
+    for (( i=1; i < $COMP_CWORD; i++ )); do
+      if [[ -n "$cmd" ]]; then
+        if [[ ! -n "$subcmd" ]]; then
+          subcmd=${COMP_WORDS[i]}
+          numargs=$(( COMP_CWORD - i - 1 ))
+        else
+          if [[ "${COMP_WORDS[i]}" == -* ]]; then
+            numoptions=$(( numoptions + 1 ))
+            numargs=$(( numargs - 1 ))
+          fi
+        fi
+      else
+        if [[ ! "${COMP_WORDS[i]}" == -* ]]; then
+          cmd=${COMP_WORDS[i]}
+        fi
+      fi
+    done
+
+    if [[ ! -n "$cmd" ]];
+    then
+      case "$cur" in
+            -*)
+              COMPREPLY=($(compgen -W "${options}" -- ${cur}))
+              return 0
+            ;;
+            *)
+              COMPREPLY=($(compgen -W "${commands}" -- ${cur}))
+              return 0
+            ;;
+        esac
+    fi
+
+    if [[ ! -n "$subcmd" ]];
+    then
+        case "$prev" in
+            "db")
+              COMPREPLY=($(compgen -W "${db_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "mirror")
+              COMPREPLY=($(compgen -W "${mirror_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "repo")
+              COMPREPLY=($(compgen -W "${repo_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "snapshot")
+              COMPREPLY=($(compgen -W "${snapshot_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "publish")
+              COMPREPLY=($(compgen -W "${publish_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "package")
+              COMPREPLY=($(compgen -W "${package_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "task")
+              COMPREPLY=($(compgen -W "${task_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "config")
+              COMPREPLY=($(compgen -W "${config_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            "api")
+              COMPREPLY=($(compgen -W "${api_subcommands}" -- ${cur}))
+              return 0
+            ;;
+            *)
+            ;;
+        esac
+    fi
+
+    case "$cmd" in
+      "mirror")
+        case "$subcmd" in
+          "create")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur}))
+                return 0
+              fi
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -with-sources -with-udebs" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force -download-limit= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "repo")
+        case "$subcmd" in
+          "add")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-force-replace -remove-files" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                _filedir '@(deb|dsc|udeb)'
+                return 0
+              ;;
+            esac
+          ;;
+          "copy"|"move")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "create")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "from" -- ${cur}))
+                return 0
+              ;;
+              2)
+                COMPREPLY=($(compgen -W "snapshot" -- ${cur}))
+                return 0
+              ;;
+              3)
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "edit")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-comment= -distribution= -component= -uploaders-file=" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "include")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
+                else
+                  comptopt -o filenames 2>/dev/null
+                  COMPREPLY=($(compgen -f -- ${cur}))
+                  return 0
+                fi
+                return 0
+              ;;
+            esac
+          ;;
+          "import")
+            case $numargs in
+              0)
+                if [[ "$cur" == -* ]]; then
+                  COMPREPLY=($(compgen -W "-with-deps -dry-run" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                fi
+                return 0
+              ;;
+              1)
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                return 0
+              ;;
+            esac
+          ;;
+          "remove")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "snapshot")
+        case "$subcmd" in
+          "create")
+            case $numargs in
+              1)
+                COMPREPLY=($(compgen -W "from empty" -- ${cur}))
+                return 0
+              ;;
+              2)
+                if [[ "$prev" == "from" ]]; then
+                  COMPREPLY=($(compgen -W "mirror repo" -- ${cur}))
+                  return 0
+                fi
+              ;;
+              3)
+                if [[ "$prev" == "mirror" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
+                  return 0
+                fi
+                if [[ "$prev" == "repo" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                  return 0
+                fi
+              ;;
+            esac
+          ;;
+          "diff")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-only-matching" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw -sort=" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "merge")
+            if [[ $numargs -gt 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-latest" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "pull")
+            if [[ $numargs -eq 0 ]] && [[ "$cur" == -* ]]; then
+              COMPREPLY=($(compgen -W "-all-matches -dry-run -no-deps -no-remove" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -lt 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "filter")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-packages" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format= -with-deps" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "rename")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "verify")
+            if [[ $numargs -eq 0 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "publish")
+        case "$subcmd" in
+          "snapshot"|"repo")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -distribution= -component= -gpg-key= -keyring= -label= -origin= -notautomatic= -butautomaticupgrades= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+              else
+                if [[ "$subcmd" == "snapshot" ]]; then
+                  COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+                else
+                  COMPREPLY=($(compgen -W "$(__aptly_repo_list)" -- ${cur}))
+                fi
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_published_prefixes)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "list")
+            if [[ $numargs -eq 0 ]]; then
+                COMPREPLY=($(compgen -W "-raw" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "update")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "switch")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-batch -force-overwrite -component= -gpg-key= -keyring= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-signing" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+
+            if [[ $numargs -ge 2 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
+              return 0
+            fi
+          ;;
+          "drop")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-force-drop" -- ${cur}))
+              else
+                COMPREPLY=($(compgen -W "$(__aptly_published_distributions)" -- ${cur}))
+              fi
+              return 0
+            fi
+
+            if [[ $numargs -eq 1 ]]; then
+              COMPREPLY=($(compgen -W "$(__aptly_prefixes_for_distribution $prev)" -- ${cur}))
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "package")
+        case "$subcmd" in
+          "search")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-format=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+          "show")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-with-files -with-references" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "serve")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "graph")
+        if [[ "$cur" == -* ]]; then
+          COMPREPLY=($(compgen -W "-format= -output=" -- ${cur}))
+          return 0
+        fi
+      ;;
+      "api")
+        case "$subcmd" in
+          "serve")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-listen=" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+      "db")
+        case "$subcmd" in
+          "cleanup")
+            if [[ $numargs -eq 0 ]]; then
+              if [[ "$cur" == -* ]]; then
+                COMPREPLY=($(compgen -W "-dry-run -verbose" -- ${cur}))
+              fi
+              return 0
+            fi
+          ;;
+        esac
+      ;;
+    esac
+} && complete -F _aptly aptly

cmd/api_serve.go

@@ -59,11 +59,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
 		os.Remove(file)
-		listener, err := net.Listen("unix", file)
+
+		var listener net.Listener
+		listener, err = net.Listen("unix", file)
 		if err != nil {
 			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
 		}
 		defer listener.Close()
+
 		err = http.Serve(listener, api.Router(context))
 		if err != nil {
 			return fmt.Errorf("unable to serve: %s", err)

cmd/cmd.go

@@ -14,6 +14,12 @@ import (
 	"github.com/smira/flag"
 )
 
+// Various command flags/UI things
+const (
+	Yes = "yes"
+	No  = "no"
+)
+
 // ListPackagesRefList shows list of packages in PackageRefList
 func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	fmt.Printf("Packages:\n")
@@ -22,26 +28,22 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 		return
 	}
 
-	err = reflist.ForEach(func(key []byte) error {
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
-		p, err2 := context.CollectionFactory().PackageCollection().ByKey(key)
-		if err2 != nil {
-			return err2
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
-	return
+	return PrintPackageList(list, "", "  ")
+
 }
 
 // PrintPackageList shows package list with specified format or default representation
-func PrintPackageList(result *deb.PackageList, format string) error {
+func PrintPackageList(result *deb.PackageList, format, prefix string) error {
+	result.PrepareIndex()
+
 	if format == "" {
-		return result.ForEach(func(p *deb.Package) error {
+		return result.ForEachIndexed(func(p *deb.Package) error {
-			context.Progress().Printf("%s\n", p)
+			context.Progress().Printf(prefix+"%s\n", p)
 			return nil
 		})
 	}
@@ -51,13 +53,13 @@ func PrintPackageList(result *deb.PackageList, format string) error {
 		return fmt.Errorf("error parsing -format template: %s", err)
 	}
 
-	return result.ForEach(func(p *deb.Package) error {
+	return result.ForEachIndexed(func(p *deb.Package) error {
 		b := &bytes.Buffer{}
 		err = formatTemplate.Execute(b, p.ExtendedStanza())
 		if err != nil {
 			return fmt.Errorf("error applying template: %s", err)
 		}
-		context.Progress().Printf("%s\n", b.String())
+		context.Progress().Printf(prefix+"%s\n", b.String())
 		return nil
 	})
 
@@ -109,12 +111,15 @@ package environment to new version.`,
 		},
 	}
 
+	cmd.Flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
 	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
 	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
 	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
 	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+	cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
 	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
 	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+	cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\" for external gpg or \"internal\" for Go internal implementation)")
 
 	if aptly.EnableDebug {
 		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")

cmd/db_cleanup.go

@@ -37,9 +37,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
@@ -67,9 +67,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		if repo.RefList() != nil {
@@ -98,9 +98,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
@@ -125,12 +125,12 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if verbose {
 			context.Progress().ColoredPrintf("- @{g}%s:%s/%s{|}", published.Storage, published.Prefix, published.Distribution)
 		}
-		if published.SourceKind != "local" {
+		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(published, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		for _, component := range published.Components() {

cmd/mirror.go

@@ -3,19 +3,19 @@ package cmd
 import (
 	"strings"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
+func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
 	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
 	keyRings := flags.Lookup("keyring").Value.Get().([]string)
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 	for _, keyRing := range keyRings {
 		verifier.AddKeyring(keyRing)
 	}

cmd/mirror_show.go

@@ -37,21 +37,21 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
 	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	downloadSources := "no"
+	downloadSources := No
 	if repo.DownloadSources {
-		downloadSources = "yes"
+		downloadSources = Yes
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
-	downloadUdebs := "no"
+	downloadUdebs := No
 	if repo.DownloadUdebs {
-		downloadUdebs = "yes"
+		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
-		filterWithDeps := "no"
+		filterWithDeps := No
 		if repo.FilterWithDeps {
-			filterWithDeps = "yes"
+			filterWithDeps = Yes
 		}
 		fmt.Printf("Filter With Deps: %s\n", filterWithDeps)
 	}

cmd/mirror_update.go

@@ -5,7 +5,9 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"sync"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/aptly/utils"
@@ -69,7 +71,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		}
 
 		var oldLen, newLen int
-		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery, context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to update: %s", err)
 		}
@@ -81,8 +83,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		queue        []deb.PackageDownloadTask
 	)
 
+	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+
 	context.Progress().Printf("Building download queue...\n")
-	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), context.CollectionFactory().PackageCollection(),
+		context.CollectionFactory().ChecksumCollection(), skipExistingPackages)
+
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -110,6 +116,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// Catch ^C
 	sigch := make(chan os.Signal)
 	signal.Notify(sigch, os.Interrupt)
+	defer signal.Stop(sigch)
+
+	abort := make(chan struct{})
+	go func() {
+		<-sigch
+		signal.Stop(sigch)
+		close(abort)
+	}()
 
 	count := len(queue)
 	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
@@ -117,37 +131,82 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	// Download from the queue
 	context.Progress().InitBar(downloadSize, true)
 
-	// Download all package files
+	downloadQueue := make(chan int)
-	ch := make(chan error, count)
 
-	// In separate goroutine (to avoid blocking main), push queue to downloader
+	var (
-	go func() {
+		errors  []string
-		for _, task := range queue {
+		errLock sync.Mutex
-			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch, maxTries)
+	)
+
+	pushError := func(err error) {
+		errLock.Lock()
+		errors = append(errors, err.Error())
+		errLock.Unlock()
 	}
 
-		// We don't need queue after this point
+	go func() {
-		queue = nil
+		for idx := range queue {
+			select {
+			case downloadQueue <- idx:
+			case <-abort:
+				return
+			}
+		}
+		close(downloadQueue)
 	}()
 
-	// Wait for all downloads to finish
+	var wg sync.WaitGroup
-	var errors []string
 
-	for count > 0 {
+	for i := 0; i < context.Config().DownloadConcurrency; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for {
 				select {
-		case <-sigch:
+				case idx, ok := <-downloadQueue:
-			signal.Stop(sigch)
+					if !ok {
+						return
+					}
+
+					task := &queue[idx]
+
+					var e error
+
+					// provision download location
+					task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+
+					// download file...
+					e = context.Downloader().DownloadWithChecksum(
+						repo.PackageURL(task.File.DownloadURL()).String(),
+						task.TempDownPath,
+						&task.File.Checksums,
+						ignoreMismatch,
+						maxTries)
+					if e != nil {
+						pushError(e)
+						continue
+					}
+				case <-abort:
+					return
+				}
+			}
+		}()
+	}
+
+	// Wait for all downloads to finish
+	wg.Wait()
+
+	select {
+	case <-abort:
 		return fmt.Errorf("unable to update: interrupted")
-		case err = <-ch:
+	default:
-			if err != nil {
-				errors = append(errors, err.Error())
-			}
-			count--
-		}
 	}
 
 	context.Progress().ShutdownBar()
-	signal.Stop(sigch)
 
 	if len(errors) > 0 {
 		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
@@ -158,7 +217,37 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	repo.FinalizeDownload()
+	// Import downloaded files
+	context.Progress().InitBar(int64(len(queue)), false)
+
+	for idx := range queue {
+
+		context.Progress().AddBar(1)
+
+		task := &queue[idx]
+
+		// and import it back to the pool
+		task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, context.CollectionFactory().ChecksumCollection())
+		if err != nil {
+			return fmt.Errorf("unable to import file: %s", err)
+		}
+
+		// update "attached" files if any
+		for _, additionalTask := range task.Additional {
+			additionalTask.File.PoolPath = task.File.PoolPath
+			additionalTask.File.Checksums = task.File.Checksums
+		}
+
+		select {
+		case <-abort:
+			return fmt.Errorf("unable to update: interrupted")
+		default:
+		}
+	}
+
+	context.Progress().ShutdownBar()
+
+	repo.FinalizeDownload(context.CollectionFactory(), context.Progress())
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -188,6 +277,7 @@ Example:
 	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

cmd/package_search.go

@@ -35,7 +35,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format)
+	PrintPackageList(result, format, "")
 
 	return err
 }

cmd/package_show.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
 	"github.com/smira/commander"
@@ -13,9 +14,9 @@ import (
 
 func printReferencesTo(p *deb.Package) (err error) {
 	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -29,9 +30,9 @@ func printReferencesTo(p *deb.Package) (err error) {
 	}
 
 	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if repo.RefList() != nil {
 			if repo.RefList().Has(p) {
@@ -45,20 +46,17 @@ func printReferencesTo(p *deb.Package) (err error) {
 	}
 
 	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		e := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 		if snapshot.RefList().Has(p) {
 			fmt.Printf("  snapshot %s\n", snapshot)
 		}
 		return nil
 	})
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }
 
 func aptlyPackageShow(cmd *commander.Command, args []string) error {
@@ -87,11 +85,18 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withFiles {
 			fmt.Printf("Files in the pool:\n")
+			packagePool := context.PackagePool()
 			for _, f := range p.Files() {
-				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				var path string
+				path, err = f.GetPoolPath(packagePool)
 				if err != nil {
 					return err
 				}
+
+				if pp, ok := packagePool.(aptly.LocalPackagePool); ok {
+					path = pp.FullPath(path)
+				}
+
 				fmt.Printf("  %s\n", path)
 			}
 			fmt.Printf("\n")

cmd/publish.go

@@ -1,17 +1,17 @@
 package cmd
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
 
-func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
+func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
-	signer := &utils.GpgSigner{}
+	signer := context.GetSigner()
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())

cmd/publish_list.go

@@ -20,9 +20,9 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 	published := make([]string, 0, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		if raw {

cmd/publish_repo.go

@@ -43,6 +43,8 @@ Example:
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("notautomatic", "", "set value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "set  value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_show.go

@@ -41,15 +41,15 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
 	fmt.Printf("Sources:\n")
 	for component, sourceID := range repo.Sources {
 		var name string
-		if repo.SourceKind == "snapshot" {
+		if repo.SourceKind == deb.SourceSnapshot {
-			source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+			source, e := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
-			if err != nil {
+			if e != nil {
 				continue
 			}
 			name = source.Name
-		} else if repo.SourceKind == "local" {
+		} else if repo.SourceKind == deb.SourceLocalRepo {
-			source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+			source, e := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
-			if err != nil {
+			if e != nil {
 				continue
 			}
 			name = source.Name

cmd/publish_snapshot.go

@@ -35,7 +35,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		message string
 	)
 
-	if cmd.Name() == "snapshot" {
+	if cmd.Name() == "snapshot" { // nolint: goconst
 		var (
 			snapshot     *deb.Snapshot
 			emptyWarning = false
@@ -71,7 +71,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		if emptyWarning {
 			context.Progress().Printf("Warning: publishing from empty source, architectures list should be complete, it can't be changed after publishing (use -architectures flag)\n")
 		}
-	} else if cmd.Name() == "repo" {
+	} else if cmd.Name() == "repo" { // nolint: goconst
 		var (
 			localRepo    *deb.LocalRepo
 			emptyWarning = false
@@ -112,12 +112,23 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 	}
 
 	distribution := context.Flags().Lookup("distribution").Value.String()
+	origin := context.Flags().Lookup("origin").Value.String()
+	notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
+	butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
 
 	published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, context.CollectionFactory())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
-	published.Origin = context.Flags().Lookup("origin").Value.String()
+	if origin != "" {
+		published.Origin = origin
+	}
+	if notAutomatic != "" {
+		published.NotAutomatic = notAutomatic
+	}
+	if butAutomaticUpgrades != "" {
+		published.ButAutomaticUpgrades = butAutomaticUpgrades
+	}
 	published.Label = context.Flags().Lookup("label").Value.String()
 
 	published.SkipContents = context.Config().SkipContentsPublishing
@@ -163,14 +174,14 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.LocalPublishedStorage); ok {
+	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
 		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
 			localStorage.PublicPath())
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
 	context.Progress().Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
-	if utils.StrSliceHasItem(published.Architectures, "source") {
+	if utils.StrSliceHasItem(published.Architectures, deb.ArchitectureSource) {
 		context.Progress().Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, repoComponents)
 	}
 	context.Progress().Printf("Don't forget to add your GPG key to apt with apt-key.\n")
@@ -211,7 +222,9 @@ Example:
 	cmd.Flag.Bool("batch", false, "run GPG with detached tty")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
-	cmd.Flag.String("origin", "", "origin name to publish")
+	cmd.Flag.String("origin", "", "overwrite origin name to publish")
+	cmd.Flag.String("notautomatic", "", "overwrite value for NotAutomatic field")
+	cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
 	cmd.Flag.String("label", "", "label to publish")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/publish_switch.go

@@ -44,7 +44,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "snapshot" {
+	if published.SourceKind != deb.SourceSnapshot {
 		return fmt.Errorf("unable to update: not a snapshot publish")
 	}
 

cmd/publish_update.go

@@ -30,7 +30,7 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	if published.SourceKind != "local" {
+	if published.SourceKind != deb.SourceLocalRepo {
 		return fmt.Errorf("unable to update: not a local repository publish")
 	}
 

cmd/repo_add.go

@@ -20,7 +20,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	name := args[0]
 
-	verifier := &utils.GpgVerifier{}
+	verifier := context.GetVerifier()
 
 	repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
 	if err != nil {
@@ -48,7 +48,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	var processedFiles, failedFiles2 []string
 
 	processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil)
+		context.CollectionFactory().PackageCollection(), &aptly.ConsoleResultReporter{Progress: context.Progress()}, nil,
+		context.CollectionFactory().ChecksumCollection())
 	failedFiles = append(failedFiles, failedFiles2...)
 	if err != nil {
 		return fmt.Errorf("unable to import package files: %s", err)
@@ -65,7 +66,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}

cmd/repo_create.go

@@ -10,7 +10,7 @@ import (
 
 func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 	var err error
-	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) {
+	if !(len(args) == 1 || (len(args) == 4 && args[1] == "from" && args[2] == "snapshot")) { // nolint: goconst
 		cmd.Usage()
 		return commander.ErrCommandError
 	}

cmd/repo_include.go

@@ -28,7 +28,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	}
 
 	if verifier == nil {
-		verifier = &utils.GpgVerifier{}
+		verifier = context.GetVerifier()
 	}
 
 	forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
@@ -97,7 +97,8 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 
 		context.Progress().Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
 
-		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
+		var repo *deb.LocalRepo
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(repoName.String())
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
@@ -131,7 +132,8 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load repo: %s", err)
 		}
 
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+		var list *deb.PackageList
+		list, err = deb.NewPackageListFromRefList(repo.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
 		if err != nil {
 			return fmt.Errorf("unable to load packages: %s", err)
 		}
@@ -151,7 +153,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 		var processedFiles2, failedFiles2 []string
 
 		processedFiles2, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			context.CollectionFactory().PackageCollection(), reporter, restriction)
+			context.CollectionFactory().PackageCollection(), reporter, restriction, context.CollectionFactory().ChecksumCollection())
 
 		if err != nil {
 			return fmt.Errorf("unable to import package files: %s", err)
@@ -184,7 +186,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 		processedFiles = utils.StrSliceDeduplicate(processedFiles)
 
 		for _, file := range processedFiles {
-			err := os.Remove(file)
+			err = os.Remove(file)
 			if err != nil {
 				return fmt.Errorf("unable to remove file: %s", err)
 			}

cmd/repo_list.go

@@ -23,9 +23,9 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+			e := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
-			if err != nil {
+			if e != nil {
-				return err
+				return e
 			}
 
 			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())

cmd/repo_move.go

@@ -34,7 +34,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		srcRepo    *deb.LocalRepo
 	)
 
-	if command == "copy" || command == "move" {
+	if command == "copy" || command == "move" { // nolint: goconst
 		srcRepo, err = context.CollectionFactory().LocalRepoCollection().ByName(args[0])
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
@@ -50,7 +50,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 
 		srcRefList = srcRepo.RefList()
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		var srcRemoteRepo *deb.RemoteRepo
 
 		srcRemoteRepo, err = context.CollectionFactory().RemoteRepoCollection().ByName(args[0])
@@ -115,18 +115,18 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	toProcess, err := srcList.Filter(queries, withDeps, dstList, context.DependencyOptions(), architecturesList)
+	toProcess, err := srcList.FilterWithProgress(queries, withDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var verb string
 
-	if command == "move" {
+	if command == "move" { // nolint: goconst
 		verb = "moved"
-	} else if command == "copy" {
+	} else if command == "copy" { // nolint: goconst
 		verb = "copied"
-	} else if command == "import" {
+	} else if command == "import" { // nolint: goconst
 		verb = "imported"
 	}
 
@@ -136,7 +136,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return err
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcList.Remove(p)
 		}
 		context.Progress().ColoredPrintf("@g[o]@| %s %s", p, verb)
@@ -156,7 +156,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
-		if command == "move" {
+		if command == "move" { // nolint: goconst
 			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
 			err = context.CollectionFactory().LocalRepoCollection().Update(srcRepo)

cmd/serve.go

@@ -60,9 +60,9 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 	published := make(map[string]*deb.PublishedRepo, context.CollectionFactory().PublishedRepoCollection().Len())
 
 	err = context.CollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
-		err := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
+		e := context.CollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.CollectionFactory())
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		sources = append(sources, repo.String())
@@ -90,13 +90,13 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
 			repo, listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 
-		if utils.StrSliceHasItem(repo.Architectures, "source") {
+		if utils.StrSliceHasItem(repo.Architectures, deb.ArchitectureSource) {
 			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
 				listenHost, listenPort, prefix, repo.Distribution, strings.Join(repo.Components(), " "))
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.LocalPublishedStorage).PublicPath()
+	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot_create.go

@@ -13,7 +13,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		snapshot *deb.Snapshot
 	)
 
-	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" { // nolint: goconst
 		// aptly snapshot create snap from mirror mirror
 		var repo *deb.RemoteRepo
 
@@ -38,7 +38,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
-	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" { // nolint: goconst
 		// aptly snapshot create snap from repo repo
 		var repo *deb.LocalRepo
 

cmd/snapshot_filter.go

@@ -66,7 +66,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	result, err := packageList.FilterWithProgress(queries, withDeps, nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}

cmd/snapshot_pull.go

@@ -96,7 +96,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 	}
 
 	// Filter with dependencies as requested
-	result, err := sourcePackageList.Filter(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList)
+	result, err := sourcePackageList.FilterWithProgress(queries, !noDeps, packageList, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}

cmd/snapshot_search.go

@@ -26,8 +26,9 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 	var reflist *deb.PackageRefList
 
-	if command == "snapshot" {
+	if command == "snapshot" { // nolint: goconst
-		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		var snapshot *deb.Snapshot
+		snapshot, err = context.CollectionFactory().SnapshotCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -39,7 +40,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 
 		reflist = snapshot.RefList()
 	} else if command == "mirror" {
-		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		var repo *deb.RemoteRepo
+		repo, err = context.CollectionFactory().RemoteRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -50,8 +52,9 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 
 		reflist = repo.RefList()
-	} else if command == "repo" {
+	} else if command == "repo" { // nolint: goconst
-		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		var repo *deb.LocalRepo
+		repo, err = context.CollectionFactory().LocalRepoCollection().ByName(name)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -99,8 +102,8 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 		}
 	}
 
-	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+	result, err := list.FilterWithProgress([]deb.PackageQuery{q}, withDeps,
-		nil, context.DependencyOptions(), architecturesList)
+		nil, context.DependencyOptions(), architecturesList, context.Progress())
 	if err != nil {
 		return fmt.Errorf("unable to search: %s", err)
 	}
@@ -110,7 +113,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format)
+	PrintPackageList(result, format, "")
 
 	return err
 }

cmd/snapshot_show.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 
+	"github.com/smira/aptly/deb"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
 )
@@ -34,20 +35,23 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
 		fmt.Printf("Sources:\n")
 		for _, sourceID := range snapshot.SourceIDs {
 			var name string
-			if snapshot.SourceKind == "snapshot" {
+			if snapshot.SourceKind == deb.SourceSnapshot {
-				source, err := context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
+				var source *deb.Snapshot
+				source, err = context.CollectionFactory().SnapshotCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
-			} else if snapshot.SourceKind == "local" {
+			} else if snapshot.SourceKind == deb.SourceLocalRepo {
-				source, err := context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
+				var source *deb.LocalRepo
+				source, err = context.CollectionFactory().LocalRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				name = source.Name
-			} else if snapshot.SourceKind == "repo" {
+			} else if snapshot.SourceKind == deb.SourceRemoteRepo {
-				source, err := context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
+				var source *deb.RemoteRepo
+				source, err = context.CollectionFactory().RemoteRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}

cmd/task_run.go

@@ -82,7 +82,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 
 	for i, command := range cmdList {
 		if !commandErrored {
-			err := context.ReOpenDatabase()
+			err = context.ReOpenDatabase()
 			if err != nil {
 				return fmt.Errorf("failed to reopen DB: %s", err)
 			}

console/progress.go

@@ -2,6 +2,7 @@ package console
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/cheggaaa/pb"
@@ -11,6 +12,7 @@ import (
 
 const (
 	codePrint = iota
+	codePrintStdErr
 	codeProgress
 	codeHideProgress
 	codeStop
@@ -28,7 +30,6 @@ type printTask struct {
 // Progress is a progress displaying subroutine, it allows to show download and other operations progress
 // mixed with progress bar
 type Progress struct {
-	stop     chan bool
 	stopped  chan bool
 	queue    chan printTask
 	bar      *pb.ProgressBar
@@ -128,6 +129,11 @@ func (p *Progress) Printf(msg string, a ...interface{}) {
 	p.queue <- printTask{code: codePrint, message: fmt.Sprintf(msg, a...)}
 }
 
+// PrintfStdErr does printf but in safe manner to stderr
+func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
+	p.queue <- printTask{code: codePrintStdErr, message: fmt.Sprintf(msg, a...)}
+}
+
 // ColoredPrintf does printf in colored way + newline
 func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
 	if RunningOnTerminal() {
@@ -183,6 +189,12 @@ func (p *Progress) worker() {
 				p.barShown = false
 			}
 			fmt.Print(task.message)
+		case codePrintStdErr:
+			if p.barShown {
+				fmt.Print("\r\033[2K")
+				p.barShown = false
+			}
+			fmt.Fprint(os.Stderr, task.message)
 		case codeProgress:
 			if hasBar {
 				fmt.Print("\r" + task.message)

context/context.go

@@ -3,6 +3,7 @@ package context
 
 import (
 	"fmt"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -17,6 +18,7 @@ import (
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/s3"
 	"github.com/smira/aptly/swift"
 	"github.com/smira/aptly/utils"
@@ -101,6 +103,9 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
+
+				// as this is fresh aptly installation, we don't need to support legacy pool locations
+				utils.Config.SkipLegacyPool = true
 				utils.SaveConfig(configLocations[0], &utils.Config)
 			}
 		}
@@ -149,6 +154,9 @@ func (context *AptlyContext) DependencyOptions() int {
 		if context.lookupOption(context.config().DepFollowSource, "dep-follow-source") {
 			context.dependencyOptions |= deb.DepFollowSource
 		}
+		if context.lookupOption(context.config().DepVerboseResolve, "dep-verbose-resolve") {
+			context.dependencyOptions |= deb.DepVerboseResolve
+		}
 	}
 
 	return context.dependencyOptions
@@ -201,8 +209,7 @@ func (context *AptlyContext) Downloader() aptly.Downloader {
 		if downloadLimit == 0 {
 			downloadLimit = context.config().DownloadLimit
 		}
-		context.downloader = http.NewDownloader(context.config().DownloadConcurrency,
+		context.downloader = http.NewDownloader(downloadLimit*1024, context._progress())
-			downloadLimit*1024, context._progress())
 	}
 
 	return context.downloader
@@ -233,13 +240,34 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 	if context.database == nil {
 		var err error
 
-		context.database, err = database.OpenDB(context.dbPath())
+		context.database, err = database.NewDB(context.dbPath())
 		if err != nil {
-			return nil, fmt.Errorf("can't open database: %s", err)
+			return nil, fmt.Errorf("can't instantiate database: %s", err)
 		}
 	}
 
-	return context.database, nil
+	tries := context.flags.Lookup("db-open-attempts").Value.Get().(int)
+	const BaseDelay = 10 * time.Second
+	const Jitter = 1 * time.Second
+
+	for ; tries >= 0; tries-- {
+		err := context.database.Open()
+		if err == nil || !strings.Contains(err.Error(), "resource temporarily unavailable") {
+			return context.database, err
+		}
+
+		if tries > 0 {
+			delay := time.Duration(rand.NormFloat64()*float64(Jitter) + float64(BaseDelay))
+			if delay < 0 {
+				delay = time.Second
+			}
+
+			context._progress().PrintfStdErr("Unable to open database, sleeping %s, attempts left %d...\n", delay, tries)
+			time.Sleep(delay)
+		}
+	}
+
+	return nil, fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
 }
 
 // CloseDatabase closes the db temporarily
@@ -256,27 +284,10 @@ func (context *AptlyContext) CloseDatabase() error {
 
 // ReOpenDatabase reopens the db after close
 func (context *AptlyContext) ReOpenDatabase() error {
-	context.Lock()
+	_, err := context.Database()
-	defer context.Unlock()
 
-	if context.database == nil {
-		return nil
-	}
-
-	const MaxTries = 10
-	const Delay = 10 * time.Second
-
-	for try := 0; try < MaxTries; try++ {
-		err := context.database.ReOpen()
-		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
 	return err
 }
-		context._progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
-		<-time.After(Delay)
-	}
-
-	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
-}
 
 // CollectionFactory builds factory producing all kinds of collections
 func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
@@ -300,7 +311,7 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	defer context.Unlock()
 
 	if context.packagePool == nil {
-		context.packagePool = files.NewPackagePool(context.config().RootDir)
+		context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
 	}
 
 	return context.packagePool
@@ -314,7 +325,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 	publishedStorage, ok := context.publishedStorages[name]
 	if !ok {
 		if name == "" {
-			publishedStorage = files.NewPublishedStorage(context.config().RootDir)
+			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
+		} else if strings.HasPrefix(name, "filesystem:") {
+			params, ok := context.config().FileSystemPublishRoots[name[11:]]
+			if !ok {
+				Fatal(fmt.Errorf("published local storage %v not configured", name[6:]))
+			}
+
+			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
 		} else if strings.HasPrefix(name, "s3:") {
 			params, ok := context.config().S3PublishRoots[name[3:]]
 			if !ok {
@@ -356,6 +374,46 @@ func (context *AptlyContext) UploadPath() string {
 	return filepath.Join(context.Config().RootDir, "upload")
 }
 
+func (context *AptlyContext) pgpProvider() string {
+	var provider string
+
+	if context.globalFlags.IsSet("gpg-provider") {
+		provider = context.globalFlags.Lookup("gpg-provider").Value.String()
+	} else {
+		provider = context.config().GpgProvider
+	}
+
+	if !(provider == "gpg" || provider == "internal") { // nolint: goconst
+		Fatal(fmt.Errorf("unknown gpg provider: %v", provider))
+	}
+
+	return provider
+}
+
+// GetSigner returns Signer with respect to provider
+func (context *AptlyContext) GetSigner() pgp.Signer {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.pgpProvider() == "gpg" { // nolint: goconst
+		return &pgp.GpgSigner{}
+	}
+
+	return &pgp.GoSigner{}
+}
+
+// GetVerifier returns Verifier with respect to provider
+func (context *AptlyContext) GetVerifier() pgp.Verifier {
+	context.Lock()
+	defer context.Unlock()
+
+	if context.pgpProvider() == "gpg" { // nolint: goconst
+		return &pgp.GpgVerifier{}
+	}
+
+	return &pgp.GoVerifier{}
+}
+
 // UpdateFlags sets internal copy of flags in the context
 func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
 	context.Lock()
@@ -406,7 +464,6 @@ func (context *AptlyContext) Shutdown() {
 		context.database = nil
 	}
 	if context.downloader != nil {
-		context.downloader.Abort()
 		context.downloader = nil
 	}
 	if context.progress != nil {
@@ -421,7 +478,6 @@ func (context *AptlyContext) Cleanup() {
 	defer context.Unlock()
 
 	if context.downloader != nil {
-		context.downloader.Shutdown()
 		context.downloader = nil
 	}
 	if context.progress != nil {

database/leveldb.go

@@ -32,8 +32,8 @@ type Storage interface {
 	ProcessByPrefix(prefix []byte, proc StorageProcessor) error
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
+	Open() error
 	Close() error
-	ReOpen() error
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
@@ -66,13 +66,19 @@ func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	return leveldb.OpenFile(path, o)
 }
 
-// OpenDB opens (creates) LevelDB database
+// NewDB creates new instance of DB, but doesn't open it (yet)
-func OpenDB(path string) (Storage, error) {
+func NewDB(path string) (Storage, error) {
-	db, err := internalOpen(path, false)
+	return &levelDB{path: path}, nil
+}
+
+// NewOpenDB creates new instance of DB and opens it
+func NewOpenDB(path string) (Storage, error) {
+	db, err := NewDB(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db, path: path}, nil
+
+	return db, db.Open()
 }
 
 // RecoverDB recovers LevelDB database from corruption
@@ -215,8 +221,8 @@ func (l *levelDB) Close() error {
 	return err
 }
 
-// Reopen tries to re-open the database
+// Reopen tries to open (re-open) the database
-func (l *levelDB) ReOpen() error {
+func (l *levelDB) Open() error {
 	if l.db != nil {
 		return nil
 	}

database/leveldb_test.go

@@ -22,7 +22,7 @@ func (s *LevelDBSuite) SetUpTest(c *C) {
 	var err error
 
 	s.path = c.MkDir()
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Assert(err, IsNil)
 }
 
@@ -46,7 +46,7 @@ func (s *LevelDBSuite) TestRecoverDB(c *C) {
 	err = RecoverDB(s.path)
 	c.Check(err, IsNil)
 
-	s.db, err = OpenDB(s.path)
+	s.db, err = NewOpenDB(s.path)
 	c.Check(err, IsNil)
 
 	result, err := s.db.Get(key)
@@ -223,7 +223,7 @@ func (s *LevelDBSuite) TestReOpen(c *C) {
 	err = s.db.Close()
 	c.Assert(err, IsNil)
 
-	err = s.db.ReOpen()
+	err = s.db.Open()
 	c.Assert(err, IsNil)
 
 	result, err := s.db.Get(key)

deb/changes.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -9,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -23,7 +25,7 @@ type Changes struct {
 	Binary                []string
 	Architectures         []string
 	Stanza                Stanza
-	SignatureKeys         []utils.GpgKey
+	SignatureKeys         []pgp.Key
 }
 
 // NewChanges moves .changes file into temporary directory and creates Changes structure
@@ -50,7 +52,7 @@ func NewChanges(path string) (*Changes, error) {
 }
 
 // VerifyAndParse does optional signature verification and parses changes files
-func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier utils.Verifier) error {
+func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier pgp.Verifier) error {
 	input, err := os.Open(filepath.Join(c.TempDir, c.ChangesName))
 	if err != nil {
 		return err
@@ -69,7 +71,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	}
 
 	if isClearSigned && !ignoreSignature {
-		keyInfo, err := verifier.VerifyClearsigned(input, false)
+		var keyInfo *pgp.KeyInfo
+		keyInfo, err = verifier.VerifyClearsigned(input, false)
 		if err != nil {
 			return err
 		}
@@ -78,7 +81,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		c.SignatureKeys = keyInfo.GoodKeys
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(input)
@@ -103,13 +106,9 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	c.Architectures = strings.Fields(c.Stanza["Architecture"])
 
 	c.Files, err = c.Files.ParseSumFields(c.Stanza)
-	if err != nil {
 	return err
 }
 
-	return nil
-}
-
 // Prepare creates temporary directory, copies file there and verifies checksums
 func (c *Changes) Prepare() error {
 	var err error
@@ -173,7 +172,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 
 	// if c.Source is empty, this would never match
 	sourceQuery := &AndQuery{
-		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"},
+		L: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource},
 		R: &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Source},
 	}
 
@@ -181,8 +180,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 	if len(c.Binary) > 0 {
 		binaryQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: c.Binary[0]}
 		// matching debug ddeb packages, they're not present in the Binary field
-		var ddebQuery PackageQuery
+		var ddebQuery PackageQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
-		ddebQuery = &FieldQuery{Field: "Name", Relation: VersionEqual, Value: fmt.Sprintf("%s-dbgsym", c.Binary[0])}
 
 		for _, binary := range c.Binary[1:] {
 			binaryQuery = &OrQuery{
@@ -206,7 +204,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		}
 
 		binaryQuery = &AndQuery{
-			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: "source"}},
+			L: &NotQuery{Q: &FieldQuery{Field: "$PackageType", Relation: VersionEqual, Value: ArchitectureSource}},
 			R: binaryQuery}
 	}
 

deb/checksum_collection.go

@@ -0,0 +1,67 @@
+package deb
+
+import (
+	"bytes"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+	"github.com/ugorji/go/codec"
+)
+
+// ChecksumCollection does management of ChecksumInfo in DB
+type ChecksumCollection struct {
+	db          database.Storage
+	codecHandle *codec.MsgpackHandle
+}
+
+// NewChecksumCollection creates new ChecksumCollection and binds it to database
+func NewChecksumCollection(db database.Storage) *ChecksumCollection {
+	return &ChecksumCollection{
+		db:          db,
+		codecHandle: &codec.MsgpackHandle{},
+	}
+}
+
+func (collection *ChecksumCollection) dbKey(path string) []byte {
+	return []byte("C" + path)
+}
+
+// Get finds checksums in DB by path
+func (collection *ChecksumCollection) Get(path string) (*utils.ChecksumInfo, error) {
+	encoded, err := collection.db.Get(collection.dbKey(path))
+	if err != nil {
+		if err == database.ErrNotFound {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	c := &utils.ChecksumInfo{}
+
+	decoder := codec.NewDecoderBytes(encoded, collection.codecHandle)
+	err = decoder.Decode(c)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+// Update adds or updates information about checksum in DB
+func (collection *ChecksumCollection) Update(path string, c *utils.ChecksumInfo) error {
+	var encodeBuffer bytes.Buffer
+
+	encoder := codec.NewEncoder(&encodeBuffer, collection.codecHandle)
+	err := encoder.Encode(c)
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Put(collection.dbKey(path), encodeBuffer.Bytes())
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &ChecksumCollection{}
+)

deb/checksum_collection_test.go

@@ -0,0 +1,47 @@
+package deb
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type ChecksumCollectionSuite struct {
+	collection *ChecksumCollection
+	c          utils.ChecksumInfo
+	db         database.Storage
+}
+
+var _ = Suite(&ChecksumCollectionSuite{})
+
+func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
+	s.c = utils.ChecksumInfo{
+		Size:   124,
+		MD5:    "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+	}
+	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.collection = NewChecksumCollection(s.db)
+}
+
+func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *ChecksumCollectionSuite) TestFlow(c *C) {
+	// checksum not stored
+	checksum, err := s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(checksum, IsNil)
+
+	// store checksum
+	err = s.collection.Update("some/path", &s.c)
+	c.Assert(err, IsNil)
+
+	// load it back
+	checksum, err = s.collection.Get("some/path")
+	c.Assert(err, IsNil)
+	c.Check(*checksum, DeepEquals, s.c)
+}

deb/collections.go

@@ -15,6 +15,7 @@ type CollectionFactory struct {
 	snapshots      *SnapshotCollection
 	localRepos     *LocalRepoCollection
 	publishedRepos *PublishedRepoCollection
+	checksums      *ChecksumCollection
 }
 
 // NewCollectionFactory creates new factory
@@ -89,6 +90,18 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 	return factory.publishedRepos
 }
 
+// ChecksumCollection returns (or creates) new ChecksumCollection
+func (factory *CollectionFactory) ChecksumCollection() *ChecksumCollection {
+	factory.Lock()
+	defer factory.Unlock()
+
+	if factory.checksums == nil {
+		factory.checksums = NewChecksumCollection(factory.db)
+	}
+
+	return factory.checksums
+}
+
 // Flush removes all references to collections, so that memory could be reclaimed
 func (factory *CollectionFactory) Flush() {
 	factory.Lock()
@@ -99,4 +112,5 @@ func (factory *CollectionFactory) Flush() {
 	factory.remoteRepos = nil
 	factory.publishedRepos = nil
 	factory.packages = nil
+	factory.checksums = nil
 }

deb/contents.go

@@ -26,8 +26,8 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
 }
 
 // Push adds package to contents index, calculating package contents as required
-func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool) error {
+func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
-	contents := p.Contents(packagePool)
+	contents := p.Contents(packagePool, progress)
 	qualifiedName := []byte(p.QualifiedName())
 
 	for _, path := range contents {

deb/deb.go

@@ -12,12 +12,20 @@ import (
 
 	"github.com/h2non/filetype/matchers"
 	"github.com/mkrautz/goar"
+	"github.com/pkg/errors"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/go-xz"
 	"github.com/smira/lzma"
 )
 
+// Source kinds
+const (
+	SourceSnapshot   = "snapshot"
+	SourceLocalRepo  = "local"
+	SourceRemoteRepo = "repo"
+)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -68,7 +76,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 }
 
 // GetControlFileFromDsc reads control file from dsc package
-func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, error) {
+func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error) {
 	file, err := os.Open(dscFile)
 	if err != nil {
 		return nil, err
@@ -82,7 +90,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 		return nil, err
 	}
 
-	var text *os.File
+	var text io.ReadCloser
 
 	if isClearSigned {
 		text, err = verifier.ExtractClearsigned(file)
@@ -105,13 +113,7 @@ func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, err
 }
 
 // GetContentsFromDeb returns list of files installed by .deb package
-func GetContentsFromDeb(packageFile string) ([]string, error) {
+func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
-	file, err := os.Open(packageFile)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
 	library := ar.NewReader(file)
 	for {
 		header, err := library.Next()
@@ -119,7 +121,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			return nil, fmt.Errorf("unable to find data.tar.* part in %s", packageFile)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("unable to read .deb archive from %s: %s", packageFile, err)
+			return nil, errors.Wrapf(err, "unable to read .deb archive from %s", packageFile)
 		}
 
 		if strings.HasPrefix(header.Name, "data.tar") {
@@ -142,7 +144,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 				} else {
 					ungzip, err := gzip.NewReader(bufReader)
 					if err != nil {
-						return nil, fmt.Errorf("unable to ungzip data.tar.gz from %s: %s", packageFile, err)
+						return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
 					}
 					defer ungzip.Close()
 					tarInput = ungzip
@@ -152,7 +154,7 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 			case "data.tar.xz":
 				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
-					return nil, fmt.Errorf("unable to unxz data.tar.xz from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
 				}
 				defer unxz.Close()
 				tarInput = unxz
@@ -172,16 +174,14 @@ func GetContentsFromDeb(packageFile string) ([]string, error) {
 					return results, nil
 				}
 				if err != nil {
-					return nil, fmt.Errorf("unable to read .tar archive from %s: %s", packageFile, err)
+					return nil, errors.Wrapf(err, "unable to read .tar archive from %s", packageFile)
 				}
 
 				if tarHeader.Typeflag == tar.TypeDir {
 					continue
 				}
 
-				if strings.HasPrefix(tarHeader.Name, "./") {
+				tarHeader.Name = strings.TrimPrefix(tarHeader.Name[2:], "./")
-					tarHeader.Name = tarHeader.Name[2:]
-				}
 				results = append(results, tarHeader.Name)
 			}
 		}

deb/deb_test.go

@@ -1,10 +1,11 @@
 package deb
 
 import (
+	"os"
 	"path/filepath"
 	"runtime"
 
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 
 	. "gopkg.in/check.v1"
 )
@@ -38,7 +39,7 @@ func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
 }
 
 func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
-	verifier := &utils.GpgVerifier{}
+	verifier := &pgp.GoVerifier{}
 
 	_, err := GetControlFileFromDsc("/no/such/file", verifier)
 	c.Check(err, ErrorMatches, ".*no such file or directory")
@@ -59,13 +60,19 @@ func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
 }
 
 func (s *DebSuite) TestGetContentsFromDeb(c *C) {
-	contents, err := GetContentsFromDeb(s.debFile)
+	f, err := os.Open(s.debFile)
+	c.Assert(err, IsNil)
+	contents, err := GetContentsFromDeb(f, s.debFile)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/share/doc/libboost-program-options-dev/changelog.gz",
 		"usr/share/doc/libboost-program-options-dev/copyright"})
+	c.Assert(f.Close(), IsNil)
 
-	contents, err = GetContentsFromDeb(s.debFile2)
+	f, err = os.Open(s.debFile2)
+	c.Assert(err, IsNil)
+	contents, err = GetContentsFromDeb(f, s.debFile2)
 	c.Check(err, IsNil)
 	c.Check(contents, DeepEquals, []string{"usr/bin/hardlink", "usr/share/man/man1/hardlink.1.gz",
 		"usr/share/doc/hardlink/changelog.gz", "usr/share/doc/hardlink/copyright", "usr/share/doc/hardlink/NEWS.Debian.gz"})
+	c.Assert(f.Close(), IsNil)
 }

deb/format.go

@@ -22,6 +22,8 @@ var (
 		"Version",
 		"Codename",
 		"Date",
+		"NotAutomatic",
+		"ButAutomaticUpgrades",
 		"Architectures",
 		"Architecture",
 		"Components",

deb/graph.go

@@ -33,9 +33,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
-		err := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -55,9 +55,9 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
-		err := collectionFactory.LocalRepoCollection().LoadComplete(repo)
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		graph.AddNode("aptly", repo.UUID, map[string]string{
@@ -81,13 +81,13 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	})
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
-		err := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-		if err != nil {
+		if e != nil {
-			return err
+			return e
 		}
 
 		description := snapshot.Description
-		if snapshot.SourceKind == "repo" {
+		if snapshot.SourceKind == SourceRemoteRepo {
 			description = "Snapshot from repo"
 		}
 
@@ -99,7 +99,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 				snapshot.Name, description, snapshot.NumPackages(), labelEnd),
 		})
 
-		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+		if snapshot.SourceKind == SourceRemoteRepo || snapshot.SourceKind == SourceLocalRepo || snapshot.SourceKind == SourceSnapshot {
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {

deb/import.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -59,8 +60,9 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
 }
 
 // ImportPackageFiles imports files into local repository
-func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier utils.Verifier,
+func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace bool, verifier pgp.Verifier,
-	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery) (processedFiles []string, failedFiles []string, err error) {
+	pool aptly.PackagePool, collection *PackageCollection, reporter aptly.ResultReporter, restriction PackageQuery,
+	checksumStorage aptly.ChecksumStorage) (processedFiles []string, failedFiles []string, err error) {
 	if forceReplace {
 		list.PrepareIndex()
 	}
@@ -116,19 +118,24 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		var files PackageFiles
+
+		if isSourcePackage {
+			files = p.Files()
+		}
+
 		var checksums utils.ChecksumInfo
 		checksums, err = utils.ChecksumsForFile(file)
 		if err != nil {
 			return nil, nil, err
 		}
 
-		if isSourcePackage {
+		mainPackageFile := PackageFile{
-			p.UpdateFiles(append(p.Files(), PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+			Filename:  filepath.Base(file),
-		} else {
+			Checksums: checksums,
-			p.UpdateFiles([]PackageFile{{Filename: filepath.Base(file), Checksums: checksums}})
 		}
 
-		err = pool.Import(file, checksums.MD5)
+		mainPackageFile.PoolPath, err = pool.Import(file, mainPackageFile.Filename, &mainPackageFile.Checksums, false, checksumStorage)
 		if err != nil {
 			reporter.Warning("Unable to import file %s into pool: %s", file, err)
 			failedFiles = append(failedFiles, file)
@@ -137,13 +144,10 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 
 		candidateProcessedFiles = append(candidateProcessedFiles, file)
 
-		// go over all files, except for the last one (.dsc/.deb itself)
+		// go over all the other files
-		for _, f := range p.Files() {
+		for i := range files {
-			if filepath.Base(f.Filename) == filepath.Base(file) {
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(files[i].Filename))
-				continue
+			files[i].PoolPath, err = pool.Import(sourceFile, files[i].Filename, &files[i].Checksums, false, checksumStorage)
-			}
-			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
-			err = pool.Import(sourceFile, f.Checksums.MD5)
 			if err != nil {
 				reporter.Warning("Unable to import file %s into pool: %s", sourceFile, err)
 				failedFiles = append(failedFiles, file)
@@ -157,6 +161,8 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
+		p.UpdateFiles(append(files, mainPackageFile))
+
 		if restriction != nil && !restriction.Matches(p) {
 			reporter.Warning("%s has been ignored as it doesn't match restriction", p)
 			failedFiles = append(failedFiles, file)

deb/index_files.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -48,7 +49,7 @@ func (file *indexFile) BufWriter() (*bufio.Writer, error) {
 	return file.w, nil
 }
 
-func (file *indexFile) Finalize(signer utils.Signer) error {
+func (file *indexFile) Finalize(signer pgp.Signer) error {
 	if file.w == nil {
 		if file.discardable {
 			return nil
@@ -155,7 +156,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 }
 
 func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
@@ -163,7 +164,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Sources")
 		} else {
 			if udeb {
@@ -188,7 +189,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ri-%s-%s-%v", component, arch, udeb)
@@ -196,7 +197,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 	if !ok {
 		var relativePath string
 
-		if arch == "source" {
+		if arch == ArchitectureSource {
 			relativePath = filepath.Join(component, "source", "Release")
 		} else {
 			if udeb {
@@ -221,7 +222,7 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 }
 
 func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *indexFile {
-	if arch == "source" {
+	if arch == ArchitectureSource {
 		udeb = false
 	}
 	key := fmt.Sprintf("ci-%s-%s-%v", component, arch, udeb)

deb/list.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"fmt"
 	"sort"
+	"strings"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
@@ -20,6 +21,8 @@ const (
 	DepFollowAllVariants
 	// DepFollowBuild pulls build dependencies
 	DepFollowBuild
+	// DepVerboseResolve emits additional logs while dependencies are being resolved
+	DepVerboseResolve
 )
 
 // PackageList is list of unique (by key) packages
@@ -31,8 +34,6 @@ const (
 type PackageList struct {
 	// Straight list of packages as map
 	packages map[string]*Package
-	// Has index been prepared?
-	indexed bool
 	// Indexed list of packages, sorted by name internally
 	packagesIndex []*Package
 	// Map of packages for each virtual package (provides)
@@ -41,6 +42,8 @@ type PackageList struct {
 	keyFunc func(p *Package) string
 	// Allow duplicates?
 	duplicatesAllowed bool
+	// Has index been prepared?
+	indexed bool
 }
 
 // PackageConflictError means that package can't be added to the list due to error
@@ -235,7 +238,7 @@ func (l *PackageList) Remove(p *Package) {
 func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	result = make([]string, 0, 10)
 	for _, pkg := range l.packages {
-		if pkg.Architecture != "all" && (pkg.Architecture != "source" || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
+		if pkg.Architecture != ArchitectureAll && (pkg.Architecture != ArchitectureSource || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
 			result = append(result, pkg.Architecture)
 		}
 	}
@@ -346,6 +349,14 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 		progress.ShutdownBar()
 	}
 
+	if options&DepVerboseResolve == DepVerboseResolve && progress != nil {
+		missingStr := make([]string, len(missing))
+		for i := range missing {
+			missingStr[i] = missing[i].String()
+		}
+		progress.ColoredPrintf("@{y}Missing dependencies:@| %s", strings.Join(missingStr, ", "))
+	}
+
 	return missing, nil
 }
 
@@ -462,6 +473,11 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
 
 // Filter filters package index by specified queries (ORed together), possibly pulling dependencies
 func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+	return l.FilterWithProgress(queries, withDependencies, source, dependencyOptions, architecturesList, nil)
+}
+
+// FilterWithProgress filters package index by specified queries (ORed together), possibly pulling dependencies and displays progress
+func (l *PackageList) FilterWithProgress(queries []PackageQuery, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string, progress aptly.Progress) (*PackageList, error) {
 	if !l.indexed {
 		panic("list not indexed, can't filter")
 	}
@@ -488,7 +504,7 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 			added = 0
 
 			// find missing dependencies
-			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, nil)
+			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, progress)
 			if err != nil {
 				return nil, err
 			}
@@ -501,9 +517,12 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 					continue
 				}
 
-				searchResults := l.Search(dep, false)
+				searchResults := l.Search(dep, true)
 				if searchResults != nil {
 					for _, p := range searchResults {
+						if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+							progress.ColoredPrintf("@{g}Injecting package@|: %s", p)
+						}
 						result.Add(p)
 						dependencySource.Add(p)
 						added++
@@ -511,6 +530,11 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 							break
 						}
 					}
+				} else {
+					if dependencyOptions&DepVerboseResolve == DepVerboseResolve && progress != nil {
+						progress.ColoredPrintf("@{r}Unsatisfied dependency@|: %s", dep.String())
+					}
+
 				}
 			}
 		}

deb/local_test.go

@@ -18,7 +18,7 @@ type LocalRepoSuite struct {
 var _ = Suite(&LocalRepoSuite{})
 
 func (s *LocalRepoSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.list = NewPackageList()
 	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
 	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
@@ -83,7 +83,7 @@ type LocalRepoCollectionSuite struct {
 var _ = Suite(&LocalRepoCollectionSuite{})
 
 func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewLocalRepoCollection(s.db)
 
 	s.list = NewPackageList()
@@ -98,14 +98,14 @@ func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("local1")
+	_, err := s.collection.ByName("local1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("local1")
+	r, err := s.collection.ByName("local1")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -116,13 +116,13 @@ func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/package.go

@@ -24,12 +24,12 @@ type Package struct {
 	Source string
 	// List of virtual packages this package provides
 	Provides []string
+	// Hash of files section
+	FilesHash uint64
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
 	IsUdeb bool
-	// Hash of files section
-	FilesHash uint64
 	// Is this >= 0.6 package?
 	V06Plus bool
 	// Offload fields
@@ -41,6 +41,20 @@ type Package struct {
 	collection *PackageCollection
 }
 
+// Package types
+const (
+	PackageTypeBinary = "deb"
+	PackageTypeUdeb   = "udeb"
+	PackageTypeSource = "source"
+)
+
+// Special arhictectures
+const (
+	ArchitectureAll    = "all"
+	ArhictectureAny    = "any"
+	ArchitectureSource = "source"
+)
+
 // Check interface
 var (
 	_ json.Marshaler = &Package{}
@@ -218,12 +232,12 @@ func (p *Package) GetField(name string) string {
 		return p.Architecture
 	case "$PackageType":
 		if p.IsSource {
-			return "source"
+			return PackageTypeSource
 		}
 		if p.IsUdeb {
-			return "udeb"
+			return PackageTypeUdeb
 		}
-		return "deb"
+		return PackageTypeBinary
 	case "Name":
 		return p.Name
 	case "Version":
@@ -256,7 +270,7 @@ func (p *Package) GetField(name string) string {
 
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
-	if p.Architecture == "all" && arch != "source" {
+	if p.Architecture == ArchitectureAll && arch != ArchitectureSource {
 		return true
 	}
 
@@ -344,7 +358,7 @@ func (p *Package) GetDependencies(options int) (dependencies []string) {
 		if source == "" {
 			source = p.Name
 		}
-		if strings.Index(source, ")") != -1 {
+		if strings.Contains(source, ")") {
 			dependencies = append(dependencies, fmt.Sprintf("%s {source}", source))
 		} else {
 			dependencies = append(dependencies, fmt.Sprintf("%s (= %s) {source}", source, p.Version))
@@ -403,32 +417,47 @@ func (p *Package) Files() PackageFiles {
 }
 
 // Contents returns cached package contents
-func (p *Package) Contents(packagePool aptly.PackagePool) []string {
+func (p *Package) Contents(packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	if p.IsSource {
 		return nil
 	}
 
-	return p.collection.loadContents(p, packagePool)
+	return p.collection.loadContents(p, packagePool, progress)
 }
 
 // CalculateContents looks up contents in package file
-func (p *Package) CalculateContents(packagePool aptly.PackagePool) []string {
+func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptly.Progress) ([]string, error) {
 	if p.IsSource {
-		return nil
+		return nil, nil
 	}
 
 	file := p.Files()[0]
-	path, err := packagePool.Path(file.Filename, file.Checksums.MD5)
+	poolPath, err := file.GetPoolPath(packagePool)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to build pool path: @| %s", err)
+		}
+		return nil, err
 	}
 
-	contents, err := GetContentsFromDeb(path)
+	reader, err := packagePool.Open(poolPath)
 	if err != nil {
-		panic(err)
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to open package in pool: @| %s", err)
+		}
+		return nil, err
+	}
+	defer reader.Close()
+
+	contents, err := GetContentsFromDeb(reader, file.Filename)
+	if err != nil {
+		if progress != nil {
+			progress.ColoredPrintf("@y[!]@| @!Failed to generate package contents: @| %s", err)
+		}
+		return nil, err
 	}
 
-	return contents
+	return contents, nil
 }
 
 // UpdateFiles saves new state of files
@@ -541,7 +570,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 	}
 
 	for i, f := range p.Files() {
-		sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+		sourcePoolPath, err := f.GetPoolPath(packagePool)
 		if err != nil {
 			return err
 		}
@@ -549,7 +578,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
-		err = publishedStorage.LinkFromPool(publishedDirectory, packagePool, sourcePath, f.Checksums.MD5, force)
+		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
 		if err != nil {
 			return err
 		}
@@ -590,29 +619,25 @@ func (p *Package) PoolDirectory() (string, error) {
 
 // PackageDownloadTask is a element of download queue for the package
 type PackageDownloadTask struct {
-	RepoURI         string
+	File         *PackageFile
-	DestinationPath string
+	Additional   []PackageDownloadTask
-	Checksums       utils.ChecksumInfo
+	TempDownPath string
 }
 
 // DownloadList returns list of missing package files for download in format
 // [[srcpath, dstpath]]
-func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
+func (p *Package) DownloadList(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result []PackageDownloadTask, err error) {
 	result = make([]PackageDownloadTask, 0, 1)
 
-	for _, f := range p.Files() {
+	files := p.Files()
-		poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+	for idx := range files {
-		if err != nil {
+		verified, err := files[idx].Verify(packagePool, checksumStorage)
-			return nil, err
-		}
-
-		verified, err := f.Verify(packagePool)
 		if err != nil {
 			return nil, err
 		}
 
 		if !verified {
-			result = append(result, PackageDownloadTask{RepoURI: f.DownloadURL(), DestinationPath: poolPath, Checksums: f.Checksums})
+			result = append(result, PackageDownloadTask{File: &files[idx]})
 		}
 	}
 
@@ -620,11 +645,11 @@ func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageD
 }
 
 // VerifyFiles verifies that all package files have neen correctly downloaded
-func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
+func (p *Package) VerifyFiles(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (result bool, err error) {
 	result = true
 
 	for _, f := range p.Files() {
-		result, err = f.Verify(packagePool)
+		result, err = f.Verify(packagePool, checksumStorage)
 		if err != nil || !result {
 			return
 		}
@@ -639,7 +664,7 @@ func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error)
 	result := make([]string, len(p.Files()))
 
 	for i, f := range p.Files() {
-		result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
+		result[i], err = f.GetPoolPath(packagePool)
 		if err != nil {
 			return nil, err
 		}

deb/package_collection.go

@@ -163,7 +163,7 @@ func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
 }
 
 // loadContents loads or calculates and saves package contents
-func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool) []string {
+func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) []string {
 	encoded, err := collection.db.Get(p.Key("xC"))
 	if err == nil {
 		contents := []string{}
@@ -181,7 +181,11 @@ func (collection *PackageCollection) loadContents(p *Package, packagePool aptly.
 		panic("unable to load contents")
 	}
 
-	contents := p.CalculateContents(packagePool)
+	contents, err := p.CalculateContents(packagePool, progress)
+	if err != nil {
+		// failed to acquire contents, don't persist it
+		return contents
+	}
 
 	var buf bytes.Buffer
 	err = codec.NewEncoder(&buf, collection.codecHandle).Encode(contents)

deb/package_collection_test.go

@@ -17,7 +17,7 @@ var _ = Suite(&PackageCollectionSuite{})
 
 func (s *PackageCollectionSuite) SetUpTest(c *C) {
 	s.p = NewPackageFromControlFile(packageStanza.Copy())
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewPackageCollection(s.db)
 }
 

deb/package_files.go

@@ -4,7 +4,6 @@ import (
 	"encoding/binary"
 	"fmt"
 	"hash/fnv"
-	"os"
 	"path/filepath"
 	"sort"
 	"strconv"
@@ -20,25 +19,33 @@ type PackageFile struct {
 	Filename string
 	// Hashes for the file
 	Checksums utils.ChecksumInfo
+	// PoolPath persists relative path to file in the package pool
+	PoolPath string
 	// Temporary field used while downloading, stored relative path on the mirror
 	downloadPath string
 }
 
 // Verify that package file is present and correct
-func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
+func (f *PackageFile) Verify(packagePool aptly.PackagePool, checksumStorage aptly.ChecksumStorage) (bool, error) {
-	poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+	generatedPoolPath, exists, err := packagePool.Verify(f.PoolPath, f.Filename, &f.Checksums, checksumStorage)
-	if err != nil {
+	if exists && err == nil {
-		return false, err
+		f.PoolPath = generatedPoolPath
 	}
 
-	st, err := os.Stat(poolPath)
+	return exists, err
-	if err != nil {
-		return false, nil
 }
 
-	// verify size
+// GetPoolPath returns path to the file in the pool
-	// TODO: verify checksum if configured
+//
-	return st.Size() == f.Checksums.Size, nil
+// For legacy packages which do not have PoolPath field set, that calculates LegacyPath via pool
+func (f *PackageFile) GetPoolPath(packagePool aptly.PackagePool) (string, error) {
+	var err error
+
+	if f.PoolPath == "" {
+		f.PoolPath, err = packagePool.LegacyPath(f.Filename, &f.Checksums)
+	}
+
+	return f.PoolPath, err
 }
 
 // DownloadURL return relative URL to package download location

deb/package_files_test.go

@@ -1,9 +1,10 @@
 package deb
 
 import (
-	"os"
+	"io/ioutil"
 	"path/filepath"
 
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/utils"
 
@@ -12,11 +13,13 @@ import (
 
 type PackageFilesSuite struct {
 	files PackageFiles
+	cs    aptly.ChecksumStorage
 }
 
 var _ = Suite(&PackageFilesSuite{})
 
 func (s *PackageFilesSuite) SetUpTest(c *C) {
+	s.cs = files.NewMockChecksumStorage()
 	s.files = PackageFiles{PackageFile{
 		Filename:     "alien-arena-common_7.40-2_i386.deb",
 		downloadPath: "pool/contrib/a/alien-arena",
@@ -29,27 +32,24 @@ func (s *PackageFilesSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageFilesSuite) TestVerify(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums.MD5)
 
-	result, err := s.files[0].Verify(packagePool)
+	result, err := s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
+	s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
 
-	result, err = s.files[0].Verify(packagePool)
+	s.files[0].Checksums.Size = 187518
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	s.files[0].Checksums.Size = 5
-	result, err = s.files[0].Verify(packagePool)
+	result, err = s.files[0].Verify(packagePool, s.cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/package_test.go

@@ -2,12 +2,11 @@ package deb
 
 import (
 	"bytes"
-	"os"
+	"io/ioutil"
 	"path/filepath"
 	"regexp"
 
 	"github.com/smira/aptly/files"
-	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
@@ -300,7 +299,7 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
 	// ~
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
-			Regexp: regexp.MustCompile("7\\.40-.*")}), Equals, true)
+			Regexp: regexp.MustCompile(`7\.40-.*`)}), Equals, true)
 	c.Check(
 		p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionRegexp, Version: "7\\.40-.*",
 			Regexp: regexp.MustCompile("40")}), Equals, true)
@@ -363,19 +362,17 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 }
 
 func (s *PackageSuite) TestLinkFromPool(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	publishedStorage := files.NewPublishedStorage(c.MkDir())
+	cs := files.NewMockChecksumStorage()
+	publishedStorage := files.NewPublishedStorage(c.MkDir(), "", "")
 	p := NewPackageFromControlFile(s.stanza)
 
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
-	c.Assert(err, IsNil)
 
-	file, err := os.Create(poolPath)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
-	c.Assert(err, IsNil)
-	file.Close()
 
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
@@ -387,7 +384,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 }
 
 func (s *PackageSuite) TestFilepathList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), true)
 	p := NewPackageFromControlFile(s.stanza)
 
 	list, err := p.FilepathList(packagePool)
@@ -396,31 +393,24 @@ func (s *PackageSuite) TestFilepathList(c *C) {
 }
 
 func (s *PackageSuite) TestDownloadList(c *C) {
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
+	cs := files.NewMockChecksumStorage()
 	p := NewPackageFromControlFile(s.stanza)
 	p.Files()[0].Checksums.Size = 5
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 
-	list, err := p.DownloadList(packagePool)
+	list, err := p.DownloadList(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
 		{
-			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
+			File: &p.Files()[0],
-			DestinationPath: poolPath,
+		},
-			Checksums: utils.ChecksumInfo{Size: 5,
+	})
-				MD5:    "1e8cba92c41420aa7baa8a5718d67122",
-				SHA1:   "46955e48cad27410a83740a21d766ce362364024",
-				SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
 
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	file, err := os.Create(poolPath)
+	list, err = p.DownloadList(packagePool, cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
-
-	list, err = p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{})
 }
@@ -428,24 +418,22 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 func (s *PackageSuite) TestVerifyFiles(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
-	packagePool := files.NewPackagePool(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir(), false)
-	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
+	cs := files.NewMockChecksumStorage()
 
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(err, IsNil)
+	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
-	file, err := os.Create(poolPath)
+	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
 
-	result, err := p.VerifyFiles(packagePool)
+	p.Files()[0].Checksums.Size = 100
+	result, err := p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, false)
 
 	p.Files()[0].Checksums.Size = 5
 
-	result, err = p.VerifyFiles(packagePool)
+	result, err = p.VerifyFiles(packagePool, cs)
 	c.Check(err, IsNil)
 	c.Check(result, Equals, true)
 }

deb/publish.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -40,13 +41,13 @@ type PublishedRepo struct {
 	Prefix               string
 	Distribution         string
 	Origin               string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
 	Label                string
 	// Architectures is a list of all architectures published
 	Architectures []string
 	// SourceKind is "local"/"repo"
 	SourceKind string
-	// Skip contents generation
-	SkipContents bool
 
 	// Map of sources by each component: component name -> source UUID
 	Sources map[string]string
@@ -55,10 +56,12 @@ type PublishedRepo struct {
 	Component string
 	// SourceUUID is UUID of either snapshot or local repo
 	SourceUUID string `codec:"SnapshotUUID"`
-
 	// Map of component to source items
 	sourceItems map[string]repoSourceItem
 
+	// Skip contents generation
+	SkipContents bool
+
 	// True if repo is being re-published
 	rePublishing bool
 }
@@ -75,6 +78,7 @@ func ParsePrefix(param string) (storage, prefix string) {
 	} else {
 		prefix = param
 	}
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
 	return
 }
 
@@ -96,19 +100,19 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 
 		if snapshot, ok := head.(*Snapshot); ok {
 			for _, uuid := range snapshot.SourceIDs {
-				if snapshot.SourceKind == "repo" {
+				if snapshot.SourceKind == SourceRemoteRepo {
 					remoteRepo, err := collectionFactory.RemoteRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, remoteRepo)
-				} else if snapshot.SourceKind == "local" {
+				} else if snapshot.SourceKind == SourceLocalRepo {
 					localRepo, err := collectionFactory.LocalRepoCollection().ByUUID(uuid)
 					if err != nil {
 						continue
 					}
 					current = append(current, localRepo)
-				} else if snapshot.SourceKind == "snapshot" {
+				} else if snapshot.SourceKind == SourceSnapshot {
 					snap, err := collectionFactory.SnapshotCollection().ByUUID(uuid)
 					if err != nil {
 						continue
@@ -166,24 +170,21 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		component               string
 		snapshot                *Snapshot
 		localRepo               *LocalRepo
-		ok                      bool
+		fields                  = make(map[string][]string)
 	)
 
 	// get first source
 	source = sources[0]
 
 	// figure out source kind
-	snapshot, ok = source.(*Snapshot)
+	switch source.(type) {
-	if ok {
+	case *Snapshot:
-		result.SourceKind = "snapshot"
+		result.SourceKind = SourceSnapshot
-	} else {
+	case *LocalRepo:
-		localRepo, ok = source.(*LocalRepo)
+		result.SourceKind = SourceLocalRepo
-		if ok {
+	default:
-			result.SourceKind = "local"
-		} else {
 		panic("unknown source kind")
 	}
-	}
 
 	for i := range sources {
 		component, source = components[i], sources[i]
@@ -213,11 +214,21 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 			return nil, fmt.Errorf("duplicate component name: %s", component)
 		}
 
-		if result.SourceKind == "snapshot" {
+		if result.SourceKind == SourceSnapshot {
 			snapshot = source.(*Snapshot)
 			result.Sources[component] = snapshot.UUID
 			result.sourceItems[component] = repoSourceItem{snapshot: snapshot}
-		} else if result.SourceKind == "local" {
+
+			if !utils.StrSliceHasItem(fields["Origin"], snapshot.Origin) {
+				fields["Origin"] = append(fields["Origin"], snapshot.Origin)
+			}
+			if !utils.StrSliceHasItem(fields["NotAutomatic"], snapshot.NotAutomatic) {
+				fields["NotAutomatic"] = append(fields["NotAutomatic"], snapshot.NotAutomatic)
+			}
+			if !utils.StrSliceHasItem(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades) {
+				fields["ButAutomaticUpgrades"] = append(fields["ButAutomaticUpgrades"], snapshot.ButAutomaticUpgrades)
+			}
+		} else if result.SourceKind == SourceLocalRepo {
 			localRepo = source.(*LocalRepo)
 			result.Sources[component] = localRepo.UUID
 			result.sourceItems[component] = repoSourceItem{localRepo: localRepo, packageRefs: localRepo.RefList()}
@@ -226,12 +237,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 
 	// clean & verify prefix
 	prefix = filepath.Clean(prefix)
-	if strings.HasPrefix(prefix, "/") {
+	prefix = strings.TrimPrefix(strings.TrimSuffix(prefix, "/"), "/")
-		prefix = prefix[1:]
-	}
-	if strings.HasSuffix(prefix, "/") {
-		prefix = prefix[:len(prefix)-1]
-	}
 	prefix = filepath.Clean(prefix)
 
 	for _, part := range strings.Split(prefix, "/") {
@@ -252,12 +258,23 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		}
 	}
 
-	if strings.Index(distribution, "/") != -1 {
+	if strings.Contains(distribution, "/") {
 		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
 	}
 
 	result.Distribution = distribution
 
+	// only fields which are unique by all given snapshots are set on published
+	if len(fields["Origin"]) == 1 {
+		result.Origin = fields["Origin"][0]
+	}
+	if len(fields["NotAutomatic"]) == 1 {
+		result.NotAutomatic = fields["NotAutomatic"][0]
+	}
+	if len(fields["ButAutomaticUpgrades"]) == 1 {
+		result.ButAutomaticUpgrades = fields["ButAutomaticUpgrades"][0]
+	}
+
 	return result, nil
 }
 
@@ -288,6 +305,8 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
 		"Distribution":         p.Distribution,
 		"Label":                p.Label,
 		"Origin":               p.Origin,
+		"NotAutomatic":         p.NotAutomatic,
+		"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
 		"Prefix":               p.Prefix,
 		"SourceKind":           p.SourceKind,
 		"Sources":              sources,
@@ -315,19 +334,27 @@ func (p *PublishedRepo) String() string {
 		sources = append(sources, fmt.Sprintf("{%s: %s}", component, source))
 	}
 
+	var extras []string
 	var extra string
 
 	if p.Origin != "" {
-		extra += fmt.Sprintf("origin: %s", p.Origin)
+		extras = append(extras, fmt.Sprintf("origin: %s", p.Origin))
+	}
+
+	if p.NotAutomatic != "" {
+		extras = append(extras, fmt.Sprintf("notautomatic: %s", p.NotAutomatic))
+	}
+
+	if p.ButAutomaticUpgrades != "" {
+		extras = append(extras, fmt.Sprintf("butautomaticupgrades: %s", p.ButAutomaticUpgrades))
 	}
 
 	if p.Label != "" {
-		if extra != "" {
+		extras = append(extras, fmt.Sprintf("label: %s", p.Label))
-			extra += ", "
-		}
-		extra += fmt.Sprintf("label: %s", p.Label)
 	}
 
+	extra = strings.Join(extras, ", ")
+
 	if extra != "" {
 		extra = " (" + extra + ")"
 	}
@@ -358,10 +385,10 @@ func (p *PublishedRepo) RefKey(component string) []byte {
 // RefList returns list of package refs in local repo
 func (p *PublishedRepo) RefList(component string) *PackageRefList {
 	item := p.sourceItems[component]
-	if p.SourceKind == "local" {
+	if p.SourceKind == SourceLocalRepo {
 		return item.packageRefs
 	}
-	if p.SourceKind == "snapshot" {
+	if p.SourceKind == SourceSnapshot {
 		return item.snapshot.RefList()
 	}
 	panic("unknown source")
@@ -380,7 +407,7 @@ func (p *PublishedRepo) Components() []string {
 
 // UpdateLocalRepo updates content from local repo in component
 func (p *PublishedRepo) UpdateLocalRepo(component string) {
-	if p.SourceKind != "local" {
+	if p.SourceKind != SourceLocalRepo {
 		panic("not local repo publish")
 	}
 
@@ -393,7 +420,7 @@ func (p *PublishedRepo) UpdateLocalRepo(component string) {
 
 // UpdateSnapshot switches snapshot for component
 func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) {
-	if p.SourceKind != "snapshot" {
+	if p.SourceKind != SourceSnapshot {
 		panic("not snapshot publish")
 	}
 
@@ -425,7 +452,7 @@ func (p *PublishedRepo) Decode(input []byte) error {
 
 	// old PublishedRepo were publishing only snapshots
 	if p.SourceKind == "" {
-		p.SourceKind = "snapshot"
+		p.SourceKind = SourceSnapshot
 	}
 
 	// <0.6 aptly used single SourceUUID + Component instead of Sources
@@ -456,7 +483,7 @@ func (p *PublishedRepo) GetLabel() string {
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
-	collectionFactory *CollectionFactory, signer utils.Signer, progress aptly.Progress, forceOverwrite bool) error {
+	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
 	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
 
 	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
@@ -474,8 +501,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 	defer func() {
-		var e error
+		e := tempDB.Close()
-		e = tempDB.Close()
 		if e != nil && progress != nil {
 			progress.Printf("failed to close temp DB: %s", err)
 		}
@@ -583,7 +609,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 							contentIndexes[key] = contentIndex
 						}
 
-						contentIndex.Push(pkg, packagePool)
+						contentIndex.Push(pkg, packagePool, progress)
 					}
 
 					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
@@ -621,7 +647,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					continue
 				}
 
-				bufWriter, err := indexes.ContentsIndex(component, arch, udeb).BufWriter()
+				var bufWriter *bufio.Writer
+				bufWriter, err = indexes.ContentsIndex(component, arch, udeb).BufWriter()
 				if err != nil {
 					return fmt.Errorf("unable to generate contents index: %v", err)
 				}
@@ -682,11 +709,17 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
+	if p.NotAutomatic != "" {
+		release["NotAutomatic"] = p.NotAutomatic
+	}
+	if p.ButAutomaticUpgrades != "" {
+		release["ButAutomaticUpgrades"] = p.ButAutomaticUpgrades
+	}
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.Distribution
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
-	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
+	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = ""
 	release["SHA1"] = ""
@@ -730,12 +763,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = indexes.RenameFiles()
+	return indexes.RenameFiles()
-	if err != nil {
-		return err
-	}
-
-	return nil
 }
 
 // RemoveFiles removes files that were created by Publish
@@ -834,7 +862,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 		return
 	}
 
-	if repo.SourceKind == "local" {
+	if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
 			err = collection.db.Put(repo.RefKey(component), item.packageRefs.Encode())
 			if err != nil {
@@ -849,7 +877,7 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) (err erro
 func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
 	repo.sourceItems = make(map[string]repoSourceItem)
 
-	if repo.SourceKind == "snapshot" {
+	if repo.SourceKind == SourceSnapshot {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -864,7 +892,7 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 			repo.sourceItems[component] = item
 		}
-	} else if repo.SourceKind == "local" {
+	} else if repo.SourceKind == SourceLocalRepo {
 		for component, sourceUUID := range repo.Sources {
 			item := repoSourceItem{}
 
@@ -932,7 +960,7 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "snapshot" {
+		if r.SourceKind == SourceSnapshot {
 			if r.SourceUUID == snapshot.UUID {
 				result = append(result, r)
 			}
@@ -952,7 +980,7 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
 	var result []*PublishedRepo
 	for _, r := range collection.list {
-		if r.SourceKind == "local" {
+		if r.SourceKind == SourceLocalRepo {
 			if r.SourceUUID == repo.UUID {
 				result = append(result, r)
 			}

deb/publish_test.go

@@ -74,6 +74,7 @@ type PublishedRepoSuite struct {
 	provider                            *FakeStorageProvider
 	publishedStorage, publishedStorage2 *files.PublishedStorage
 	packagePool                         aptly.PackagePool
+	cs                                  aptly.ChecksumStorage
 	localRepo                           *LocalRepo
 	snapshot, snapshot2                 *Snapshot
 	db                                  database.Storage
@@ -86,17 +87,31 @@ var _ = Suite(&PublishedRepoSuite{})
 func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
 
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
 		"":            s.publishedStorage,
 		"files:other": s.publishedStorage2}}
-	s.packagePool = files.NewPackagePool(s.root)
+	s.packagePool = files.NewPackagePool(s.root, false)
+	s.cs = files.NewMockChecksumStorage()
+
+	tmpFilepath := filepath.Join(c.MkDir(), "file")
+	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+
+	var err error
+	s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs)
+	c.Assert(err, IsNil)
+
+	s.p1.UpdateFiles(s.p1.Files())
+	s.p2.UpdateFiles(s.p1.Files())
+	s.p3.UpdateFiles(s.p1.Files())
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
@@ -131,12 +146,6 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	s.repo5.SkipContents = true
-
-	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
-	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
-	f, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	f.Close()
 }
 
 func (s *PublishedRepoSuite) TearDownTest(c *C) {
@@ -268,7 +277,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
 	c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
 
 	s.localRepo.DefaultDistribution = "precise"
@@ -292,7 +301,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "squeeze")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
 
-	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
+	_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
 }
 
@@ -440,7 +449,7 @@ type PublishedRepoCollectionSuite struct {
 var _ = Suite(&PublishedRepoCollectionSuite{})
 
 func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -468,7 +477,7 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
-	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	_, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
@@ -480,7 +489,7 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(s.collection.Add(s.repo4), IsNil)
 	c.Assert(s.collection.Add(s.repo5), IsNil)
 
-	r, err = s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
+	r, err := s.collection.ByStoragePrefixDistribution("", "ppa", "anaconda")
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -496,16 +505,17 @@ func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C)
 	c.Assert(r.String(), Equals, s.repo1.String())
 
 	r, err = s.collection.ByStoragePrefixDistribution("files:other", "ppa", "precise")
+	c.Assert(err, IsNil)
 	c.Check(r.String(), Equals, s.repo5.String())
 }
 
 func (s *PublishedRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID(s.repo1.UUID)
+	_, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.repo1), IsNil)
 
-	r, err = s.collection.ByUUID(s.repo1.UUID)
+	r, err := s.collection.ByUUID(s.repo1.UUID)
 	c.Assert(err, IsNil)
 
 	err = s.collection.LoadComplete(r, s.factory)
@@ -552,7 +562,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 		Prefix:        "ppa",
 		Distribution:  "anaconda",
 		Architectures: []string{"i386"},
-		SourceKind:    "local",
+		SourceKind:    SourceLocalRepo,
 		Component:     "contrib",
 		SourceUUID:    s.localRepo.UUID,
 	}
@@ -630,7 +640,7 @@ type PublishedRepoRemoveSuite struct {
 var _ = Suite(&PublishedRepoRemoveSuite{})
 
 func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.factory = NewCollectionFactory(s.db)
 
 	s.snapshotCollection = s.factory.SnapshotCollection()
@@ -653,7 +663,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
-	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
 	s.publishedStorage.MkDir("ppa/dists/anaconda")
 	s.publishedStorage.MkDir("ppa/dists/meduza")
 	s.publishedStorage.MkDir("ppa/dists/osminog")
@@ -663,7 +673,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.publishedStorage.MkDir("pool/main")
 
 	s.root2 = c.MkDir()
-	s.publishedStorage2 = files.NewPublishedStorage(s.root2)
+	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
 	s.publishedStorage2.MkDir("ppa/dists/osminog")
 	s.publishedStorage2.MkDir("ppa/pool/contrib")
 

deb/query.go

@@ -204,7 +204,7 @@ func (q *FieldQuery) Fast(list PackageCatalog) bool {
 // String interface
 func (q *FieldQuery) String() string {
 	escape := func(val string) string {
-		if strings.IndexAny(val, "()|,!{} \t\n") != -1 {
+		if strings.ContainsAny(val, "()|,!{} \t\n") {
 			return "'" + strings.Replace(strings.Replace(val, "\\", "\\\\", -1), "'", "\\'", -1) + "'"
 		}
 		return val

deb/reflist.go

@@ -92,7 +92,7 @@ func (l *PackageRefList) Has(p *Package) bool {
 	key := p.Key("")
 
 	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
-	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+	return i < len(l.Refs) && bytes.Equal(l.Refs[i], key)
 }
 
 // Strings builds list of strings with package keys
@@ -395,6 +395,4 @@ func (l *PackageRefList) FilterLatestRefs() {
 
 		lastArch, lastName, lastVer = arch, name, ver
 	}
-
-	return
 }

deb/reflist_test.go

@@ -44,7 +44,7 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 	coll.Update(s.p1)
 	coll.Update(s.p3)
@@ -166,7 +166,7 @@ func (s *PackageRefListSuite) TestSubstract(c *C) {
 }
 
 func (s *PackageRefListSuite) TestDiff(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{
@@ -238,7 +238,7 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 }
 
 func (s *PackageRefListSuite) TestMerge(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
+	db, _ := database.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
 
 	packages := []*Package{

deb/remote.go

@@ -7,7 +7,6 @@ import (
 	"net/url"
 	"os"
 	"path"
-	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -18,6 +17,7 @@ import (
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-uuid/uuid"
 	"github.com/ugorji/go/codec"
@@ -45,10 +45,6 @@ type RemoteRepo struct {
 	Components []string
 	// List of architectures to fetch, if empty, then fetch all architectures
 	Architectures []string
-	// Should we download sources?
-	DownloadSources bool
-	// Should we download .udebs?
-	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
@@ -57,20 +53,22 @@ type RemoteRepo struct {
 	ReleaseFiles map[string]utils.ChecksumInfo
 	// Filter for packages
 	Filter string
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
 	// SkipComponentCheck skips component list verification
 	SkipComponentCheck bool
 	// SkipArchitectureCheck skips architecture list verification
 	SkipArchitectureCheck bool
-	// Status marks state of repository (being updated, no action)
+	// Should we download sources?
-	Status int
+	DownloadSources bool
-	// WorkerPID is PID of the process modifying the mirror (if any)
+	// Should we download .udebs?
-	WorkerPID int
+	DownloadUdebs bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
-	// Temporary list of package refs
-	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
 	// Current list of packages (filled while updating mirror)
@@ -193,49 +191,49 @@ func (repo *RemoteRepo) CheckLock() error {
 	return nil
 }
 
-// ReleaseURL returns URL to Release* files in repo root
+// IndexesRootURL builds URL for various indexes
-func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
+func (repo *RemoteRepo) IndexesRootURL() *url.URL {
 	var path *url.URL
 
 	if !repo.IsFlat() {
-		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
+		path = &url.URL{Path: fmt.Sprintf("dists/%s/", repo.Distribution)}
 	} else {
-		path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
+		path = &url.URL{Path: repo.Distribution}
 	}
 
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// FlatBinaryURL returns URL to Packages files for flat repo
+// ReleaseURL returns URL to Release* files in repo root
-func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
+func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
+	return repo.IndexesRootURL().ResolveReference(&url.URL{Path: name})
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// FlatSourcesURL returns URL to Sources files for flat repo
+// FlatBinaryPath returns path to Packages files for flat repo
-func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
+func (repo *RemoteRepo) FlatBinaryPath() string {
-	path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
+	return "Packages"
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// BinaryURL returns URL of Packages files for given component and
+// FlatSourcesPath returns path to Sources files for flat repo
+func (repo *RemoteRepo) FlatSourcesPath() string {
+	return "Sources"
+}
+
+// BinaryPath returns path to Packages files for given component and
 // architecture
-func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
+func (repo *RemoteRepo) BinaryPath(component string, architecture string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return fmt.Sprintf("%s/binary-%s/Packages", component, architecture)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// SourcesURL returns URL of Sources files for given component
+// SourcesPath returns path to Sources files for given component
-func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
+func (repo *RemoteRepo) SourcesPath(component string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/source/Sources", repo.Distribution, component)}
+	return fmt.Sprintf("%s/source/Sources", component)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// UdebURL returns URL of Packages files for given component and
+// UdebPath returns path of Packages files for given component and
 // architecture
-func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
+func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
-	return repo.archiveRootURL.ResolveReference(path)
 }
 
 // PackageURL returns URL of package file relative to repository root
@@ -246,7 +244,7 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 }
 
 // Fetch updates information about repository
-func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
+func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
 	var (
 		release, inrelease, releasesig *os.File
 		err                            error
@@ -292,7 +290,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
 			return err
 		}
 
-		err = verifier.VerifyDetachedSignature(releasesig, release)
+		err = verifier.VerifyDetachedSignature(releasesig, release, true)
 		if err != nil {
 			return err
 		}
@@ -316,7 +314,7 @@ ok:
 		architectures := strings.Split(stanza["Architectures"], " ")
 		sort.Strings(architectures)
 		// "source" architecture is never present, despite Release file claims
-		architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
+		architectures = utils.StrSlicesSubstract(architectures, []string{ArchitectureSource})
 		if len(repo.Architectures) == 0 {
 			repo.Architectures = architectures
 		} else if !repo.SkipArchitectureCheck {
@@ -331,9 +329,7 @@ ok:
 		if strings.Contains(repo.Distribution, "/") {
 			distributionLast := path.Base(repo.Distribution) + "/"
 			for i := range components {
-				if strings.HasPrefix(components[i], distributionLast) {
+				components[i] = strings.TrimPrefix(components[i], distributionLast)
-					components[i] = components[i][len(distributionLast):]
-				}
 			}
 		}
 		if len(repo.Components) == 0 {
@@ -411,30 +407,30 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
-	packagesURLs := [][]string{}
+	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
 		if repo.DownloadSources {
-			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
 				if repo.DownloadUdebs {
-					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
 				}
 			}
 			if repo.DownloadSources {
-				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
 			}
 		}
 	}
 
-	for _, info := range packagesURLs {
+	for _, info := range packagesPaths {
-		url, kind := info[0], info[1]
+		path, kind := info[0], info[1]
-		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
+		packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
 		if err != nil {
 			return err
 		}
@@ -459,11 +455,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 
 			var p *Package
 
-			if kind == "binary" {
+			if kind == PackageTypeBinary {
 				p = NewPackageFromControlFile(stanza)
-			} else if kind == "udeb" {
+			} else if kind == PackageTypeUdeb {
 				p = NewUdebPackageFromControlFile(stanza)
-			} else if kind == "source" {
+			} else if kind == PackageTypeSource {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
@@ -477,11 +473,6 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					return err
 				}
 			}
-
-			err = collectionFactory.PackageCollection().Update(p)
-			if err != nil {
-				return err
-			}
 		}
 
 		progress.ShutdownBar()
@@ -491,14 +482,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 }
 
 // ApplyFilter applies filtering to already built PackageList
-func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery, progress aptly.Progress) (oldLen, newLen int, err error) {
 	repo.packageList.PrepareIndex()
 
 	emptyList := NewPackageList()
 	emptyList.PrepareIndex()
 
 	oldLen = repo.packageList.Len()
-	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress)
 	if repo.packageList != nil {
 		newLen = repo.packageList.Len()
 	}
@@ -506,24 +497,40 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
 }
 
 // BuildDownloadQueue builds queue, discards current PackageList
-func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
 	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
-	seen := make(map[string]struct{}, repo.packageList.Len())
+	seen := make(map[string]int, repo.packageList.Len())
 
 	err = repo.packageList.ForEach(func(p *Package) error {
-		list, err2 := p.DownloadList(packagePool)
+		if repo.packageRefs != nil && skipExistingPackages {
+			if repo.packageRefs.Has(p) {
+				// skip this package, but load checksums/files from package in DB
+				var prevP *Package
+				prevP, err = packageCollection.ByKey(p.Key(""))
+				if err != nil {
+					return err
+				}
+
+				p.UpdateFiles(prevP.Files())
+				return nil
+			}
+		}
+
+		list, err2 := p.DownloadList(packagePool, checksumStorage)
 		if err2 != nil {
 			return err2
 		}
-		p.files = nil
 
 		for _, task := range list {
-			key := task.RepoURI + "-" + task.DestinationPath
+			key := task.File.DownloadURL()
-			_, found := seen[key]
+			idx, found := seen[key]
 			if !found {
 				queue = append(queue, task)
-				downloadSize += task.Checksums.Size
+				downloadSize += task.File.Checksums.Size
-				seen[key] = struct{}{}
+				seen[key] = len(queue) - 1
+			} else {
+				// hook up the task to duplicate entry already on the list
+				queue[idx].Additional = append(queue[idx].Additional, task)
 			}
 		}
 
@@ -533,17 +540,39 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue
 		return
 	}
 
-	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
-	// free up package list, we don't need it after this point
-	repo.packageList = nil
-
 	return
 }
 
 // FinalizeDownload swaps for final value of package refs
-func (repo *RemoteRepo) FinalizeDownload() {
+func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
 	repo.LastDownloadDate = time.Now()
-	repo.packageRefs = repo.tempPackageRefs
+
+	if progress != nil {
+		progress.InitBar(int64(repo.packageList.Len()), true)
+	}
+
+	var i int
+
+	// update all the packages in collection
+	err := repo.packageList.ForEach(func(p *Package) error {
+		i++
+		if progress != nil {
+			progress.SetBar(i)
+		}
+		// download process might have updated checksums
+		p.UpdateFiles(p.Files())
+		return collectionFactory.PackageCollection().Update(p)
+	})
+
+	repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
+
+	if progress != nil {
+		progress.ShutdownBar()
+	}
+
+	repo.packageList = nil
+
+	return err
 }
 
 // Encode does msgpack encoding of RemoteRepo

deb/remote_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/files"
 	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
@@ -27,11 +28,11 @@ func (n *NullVerifier) InitKeyring() error {
 func (n *NullVerifier) AddKeyring(keyring string) {
 }
 
-func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) error {
+func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader, hint bool) error {
 	return nil
 }
 
-func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*utils.GpgKeyInfo, error) {
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader, hint bool) (*pgp.KeyInfo, error) {
 	return nil, nil
 }
 
@@ -81,6 +82,7 @@ type RemoteRepoSuite struct {
 	db                database.Storage
 	collectionFactory *CollectionFactory
 	packagePool       aptly.PackagePool
+	cs                aptly.ChecksumStorage
 }
 
 var _ = Suite(&RemoteRepoSuite{})
@@ -90,9 +92,10 @@ func (s *RemoteRepoSuite) SetUpTest(c *C) {
 	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collectionFactory = NewCollectionFactory(s.db)
-	s.packagePool = files.NewPackagePool(c.MkDir())
+	s.packagePool = files.NewPackagePool(c.MkDir(), false)
+	s.cs = files.NewMockChecksumStorage()
 	s.SetUpPackages()
 	s.progress.Start()
 }
@@ -155,24 +158,30 @@ func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
 	c.Assert(s.flat.ReleaseURL("Release").String(), Equals, "http://repos.express42.com/virool/precise/Release")
 }
 
-func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
+func (s *RemoteRepoSuite) TestIndexesRootURL(c *C) {
-	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
+	c.Assert(s.repo.IndexesRootURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/")
+
+	c.Assert(s.flat.IndexesRootURL().String(), Equals, "http://repos.express42.com/virool/precise/")
 }
 
-func (s *RemoteRepoSuite) TestUdebURL(c *C) {
+func (s *RemoteRepoSuite) TestBinaryPath(c *C) {
-	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+	c.Assert(s.repo.BinaryPath("main", "amd64"), Equals, "main/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
+func (s *RemoteRepoSuite) TestUdebPath(c *C) {
-	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
+	c.Assert(s.repo.UdebPath("main", "amd64"), Equals, "main/debian-installer/binary-amd64/Packages")
 }
 
-func (s *RemoteRepoSuite) TestFlatBinaryURL(c *C) {
+func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
-	c.Assert(s.flat.FlatBinaryURL().String(), Equals, "http://repos.express42.com/virool/precise/Packages")
+	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
-func (s *RemoteRepoSuite) TestFlatSourcesURL(c *C) {
+func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
-	c.Assert(s.flat.FlatSourcesURL().String(), Equals, "http://repos.express42.com/virool/precise/Sources")
+	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
+}
+
+func (s *RemoteRepoSuite) TestFlatSourcesPath(c *C) {
+	c.Assert(s.flat.FlatSourcesPath(), Equals, "Sources")
 }
 
 func (s *RemoteRepoSuite) TestPackageURL(c *C) {
@@ -266,18 +275,62 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
@@ -298,13 +351,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
-	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -312,7 +366,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.repo.FinalizeDownload()
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
@@ -323,6 +377,56 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+
+	err = s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err = s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
@@ -340,18 +444,64 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(3))
 	c.Check(queue, HasLen, 1)
-	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].File.DownloadURL(), Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
@@ -375,13 +525,14 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
-	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
 	c.Check(size, Equals, int64(15))
 	c.Check(queue, HasLen, 4)
 
 	q := make([]string, 4)
 	for i := range q {
-		q[i] = queue[i].RepoURI
+		q[i] = queue[i].File.DownloadURL()
 	}
 	sort.Strings(q)
 	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
@@ -389,7 +540,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
 	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
 
-	s.flat.FinalizeDownload()
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
@@ -401,6 +552,58 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	c.Assert(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
+
+	// Next call must return an empty download list with option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, true)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(0))
+	c.Check(queue, HasLen, 0)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	// Next call must return the download list without option "skip-existing-packages"
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", &http.Error{Code: 404})
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.xz", &http.Error{Code: 404})
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+
+	err = s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err = s.flat.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	s.flat.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.flat.packageRefs, NotNil)
 }
 
 type RemoteRepoCollectionSuite struct {
@@ -412,7 +615,7 @@ type RemoteRepoCollectionSuite struct {
 var _ = Suite(&RemoteRepoCollectionSuite{})
 
 func (s *RemoteRepoCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewRemoteRepoCollection(s.db)
 	s.SetUpPackages()
 }
@@ -422,14 +625,14 @@ func (s *RemoteRepoCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
-	r, err := s.collection.ByName("yandex")
+	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
-	r, err = s.collection.ByName("yandex")
+	r, err := s.collection.ByName("yandex")
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 
@@ -440,13 +643,13 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
-	r, err := s.collection.ByUUID("some-uuid")
+	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
-	r, err = s.collection.ByUUID(repo.UUID)
+	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }

deb/snapshot.go

@@ -31,6 +31,10 @@ type Snapshot struct {
 	// Description of how snapshot was created
 	Description string
 
+	Origin               string
+	NotAutomatic         string
+	ButAutomaticUpgrades string
+
 	packageRefs *PackageRefList
 }
 
@@ -44,28 +48,33 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 		UUID:                 uuid.New(),
 		Name:                 name,
 		CreatedAt:            time.Now(),
-		SourceKind:  "repo",
+		SourceKind:           SourceRemoteRepo,
 		SourceIDs:            []string{repo.UUID},
 		Description:          fmt.Sprintf("Snapshot from mirror %s", repo),
+		Origin:               repo.Meta["Origin"],
+		NotAutomatic:         repo.Meta["NotAutomatic"],
+		ButAutomaticUpgrades: repo.Meta["ButAutomaticUpgrades"],
 		packageRefs:          repo.packageRefs,
 	}, nil
 }
 
 // NewSnapshotFromLocalRepo creates snapshot from current state of local repository
 func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
-	if repo.packageRefs == nil {
+	snap := &Snapshot{
-		return nil, errors.New("local repo doesn't have packages")
-	}
-
-	return &Snapshot{
 		UUID:        uuid.New(),
 		Name:        name,
 		CreatedAt:   time.Now(),
-		SourceKind:  "local",
+		SourceKind:  SourceLocalRepo,
 		SourceIDs:   []string{repo.UUID},
 		Description: fmt.Sprintf("Snapshot from local repo %s", repo),
 		packageRefs: repo.packageRefs,
-	}, nil
+	}
+
+	if snap.packageRefs == nil {
+		snap.packageRefs = NewPackageRefList()
+	}
+
+	return snap, nil
 }
 
 // NewSnapshotFromPackageList creates snapshot from PackageList
@@ -255,7 +264,7 @@ func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Sn
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "repo" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}
@@ -267,7 +276,7 @@ func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snap
 	var result []*Snapshot
 
 	for _, s := range collection.list {
-		if s.SourceKind == "local" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
 			result = append(result, s)
 		}
 	}

deb/snapshot_test.go

@@ -26,7 +26,7 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
-	c.Check(snapshot.SourceKind, Equals, "repo")
+	c.Check(snapshot.SourceKind, Equals, SourceRemoteRepo)
 	c.Check(snapshot.SourceIDs, DeepEquals, []string{s.repo.UUID})
 
 	s.repo.packageRefs = nil
@@ -37,11 +37,17 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 func (s *SnapshotSuite) TestNewSnapshotFromLocalRepo(c *C) {
 	localRepo := NewLocalRepo("lala", "hoorah!")
 
-	_, err := NewSnapshotFromLocalRepo("snap2", localRepo)
+	snapshot, err := NewSnapshotFromLocalRepo("snap2", localRepo)
-	c.Check(err, ErrorMatches, "local repo doesn't have packages")
+	c.Assert(err, IsNil)
+	c.Check(snapshot.Name, Equals, "snap2")
+	c.Check(snapshot.NumPackages(), Equals, 0)
+	c.Check(snapshot.RefList().Len(), Equals, 0)
+	c.Check(snapshot.SourceKind, Equals, "local")
+	c.Check(snapshot.SourceIDs, DeepEquals, []string{localRepo.UUID})
 
 	localRepo.UpdateRefList(s.reflist)
-	snapshot, _ := NewSnapshotFromLocalRepo("snap1", localRepo)
+	snapshot, err = NewSnapshotFromLocalRepo("snap1", localRepo)
+	c.Assert(err, IsNil)
 	c.Check(snapshot.Name, Equals, "snap1")
 	c.Check(snapshot.NumPackages(), Equals, 3)
 	c.Check(snapshot.RefList().Len(), Equals, 3)
@@ -106,7 +112,7 @@ type SnapshotCollectionSuite struct {
 var _ = Suite(&SnapshotCollectionSuite{})
 
 func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
-	s.db, _ = database.OpenDB(c.MkDir())
+	s.db, _ = database.NewOpenDB(c.MkDir())
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
@@ -132,7 +138,7 @@ func (s *SnapshotCollectionSuite) TearDownTest(c *C) {
 }
 
 func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
-	snapshot, err := s.collection.ByName("snap1")
+	_, err := s.collection.ByName("snap1")
 	c.Assert(err, ErrorMatches, "*.not found")
 
 	c.Assert(s.collection.Add(s.snapshot1), IsNil)
@@ -140,7 +146,7 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 
 	c.Assert(s.collection.Add(s.snapshot2), IsNil)
 
-	snapshot, err = s.collection.ByName("snap1")
+	snapshot, err := s.collection.ByName("snap1")
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.String(), Equals, s.snapshot1.String())
 

deb/uploaders.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/DisposaBoy/JsonConfigReader"
+	"github.com/smira/aptly/pgp"
 	"github.com/smira/aptly/utils"
 )
 
@@ -85,7 +86,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			deny := u.ExpandGroups(rule.Deny)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range deny {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return fmt.Errorf("denied according to rule: %s", rule)
 					}
 				}
@@ -94,7 +95,7 @@ func (u *Uploaders) IsAllowed(changes *Changes) error {
 			allow := u.ExpandGroups(rule.Allow)
 			for _, key := range changes.SignatureKeys {
 				for _, item := range allow {
-					if item == "*" || key.Matches(utils.GpgKey(item)) {
+					if item == "*" || key.Matches(pgp.Key(item)) {
 						return nil
 					}
 				}

deb/uploaders_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	"github.com/smira/aptly/utils"
+	"github.com/smira/aptly/pgp"
 	. "gopkg.in/check.v1"
 )
 
@@ -58,24 +58,24 @@ func (s *UploadersSuite) TestIsAllowed(c *C) {
 	}
 
 	// no keys - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{}, Stanza: Stanza{"Source": "calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// no rule - not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1", "EC4B033C70096AD1"}, Stanza: Stanza{"Source": "unknown-calamares"}}), ErrorMatches, "denied as no rule matches")
 
 	// first rule: allow anyone do stuff with calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "calamares"}}), IsNil)
 
 	// second rule: nobody is allowed to do stuff with never-calamares
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "1234ABCD"}, Stanza: Stanza{"Source": "never-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"\\*\"\\]}")
 
 	// third rule: anyone from the group or explicit key
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"45678901", "12345678"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"37E1C17570096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"70096AD1"}, Stanza: Stanza{"Source": "some-calamares"}}), IsNil)
 
 	// fourth rule: some are not allowed
-	c.Check(u.IsAllowed(&Changes{SignatureKeys: []utils.GpgKey{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
+	c.Check(u.IsAllowed(&Changes{SignatureKeys: []pgp.Key{"ABCD1234", "45678901"}, Stanza: Stanza{"Source": "some-calamares"}}),
 		ErrorMatches, "denied according to rule: {\"condition\":\"\",\"allow\":null,\"deny\":\\[\"45678901\",\"12345678\"\\]}")
 }

files/mocks.go

@@ -0,0 +1,36 @@
+package files
+
+import (
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+)
+
+type mockChecksumStorage struct {
+	store map[string]utils.ChecksumInfo
+}
+
+// NewMockChecksumStorage creates aptly.ChecksumStorage for tests
+func NewMockChecksumStorage() aptly.ChecksumStorage {
+	return &mockChecksumStorage{
+		store: make(map[string]utils.ChecksumInfo),
+	}
+}
+
+func (st *mockChecksumStorage) Get(path string) (*utils.ChecksumInfo, error) {
+	c, ok := st.store[path]
+	if !ok {
+		return nil, nil
+	}
+
+	return &c, nil
+}
+
+func (st *mockChecksumStorage) Update(path string, c *utils.ChecksumInfo) error {
+	st.store[path] = *c
+	return nil
+}
+
+// Check interface
+var (
+	_ aptly.ChecksumStorage = &mockChecksumStorage{}
+)

files/package_pool.go

@@ -7,33 +7,51 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"syscall"
+
+	"github.com/smira/go-uuid/uuid"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // PackagePool is deduplicated storage of package files on filesystem
 type PackagePool struct {
 	sync.Mutex
+
 	rootPath           string
+	supportLegacyPaths bool
 }
 
 // Check interface
 var (
 	_ aptly.PackagePool      = (*PackagePool)(nil)
+	_ aptly.LocalPackagePool = (*PackagePool)(nil)
 )
 
 // NewPackagePool creates new instance of PackagePool which specified root
-func NewPackagePool(root string) *PackagePool {
+func NewPackagePool(root string, supportLegacyPaths bool) *PackagePool {
-	return &PackagePool{rootPath: filepath.Join(root, "pool")}
+	rootPath := filepath.Join(root, "pool")
+	rootPath, err := filepath.Abs(rootPath)
+	if err != nil {
+		panic(err)
 	}
 
-// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+	return &PackagePool{
-func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
+		rootPath:           rootPath,
+		supportLegacyPaths: supportLegacyPaths,
+	}
+}
+
+// LegacyPath returns path relative to pool's root for pre-1.1 aptly (based on MD5)
+func (pool *PackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
 	filename = filepath.Base(filename)
 	if filename == "." || filename == "/" {
 		return "", fmt.Errorf("filename %s is invalid", filename)
 	}
 
+	hashMD5 := checksums.MD5
+
 	if len(hashMD5) < 4 {
 		return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
 	}
@@ -41,14 +59,21 @@ func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string,
 	return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
 }
 
-// Path returns full path to package file in pool given any name and hash of file contents
+// buildPoolPath generates pool path based on file checksum
-func (pool *PackagePool) Path(filename string, hashMD5 string) (string, error) {
+func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
-	relative, err := pool.RelativePath(filename, hashMD5)
+	filename = filepath.Base(filename)
-	if err != nil {
+	if filename == "." || filename == "/" {
-		return "", err
+		return "", fmt.Errorf("filename %s is invalid", filename)
 	}
 
-	return filepath.Join(pool.rootPath, relative), nil
+	hash := checksums.SHA256
+
+	if len(hash) < 4 {
+		// this should never happen in real life
+		return "", fmt.Errorf("unable to compute pool location for filename %v, SHA256 is missing", filename)
+	}
+
+	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename), nil
 }
 
 // FilepathList returns file paths of all the files in the pool
@@ -113,57 +138,276 @@ func (pool *PackagePool) Remove(path string) (size int64, err error) {
 	return info.Size(), err
 }
 
+func (pool *PackagePool) ensureChecksums(poolPath, fullPoolPath string, checksumStorage aptly.ChecksumStorage) (targetChecksums *utils.ChecksumInfo, err error) {
+	targetChecksums, err = checksumStorage.Get(poolPath)
+	if err != nil {
+		return
+	}
+
+	if targetChecksums == nil {
+		// we don't have checksums stored yet for this file
+		targetChecksums = &utils.ChecksumInfo{}
+		*targetChecksums, err = utils.ChecksumsForFile(fullPoolPath)
+		if err != nil {
+			return
+		}
+
+		err = checksumStorage.Update(poolPath, targetChecksums)
+	}
+
+	return
+}
+
+// Verify checks whether file exists in the pool and fills back checksum info
+//
+// if poolPath is empty, poolPath is generated automatically based on checksum info (if available)
+// in any case, if function returns true, it also fills back checksums with complete information about the file in the pool
+func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	possiblePoolPaths := []string{}
+
+	if poolPath != "" {
+		possiblePoolPaths = append(possiblePoolPaths, poolPath)
+	} else {
+		// try to guess
+		if checksums.SHA256 != "" {
+			modernPath, err := pool.buildPoolPath(basename, checksums)
+			if err != nil {
+				return "", false, err
+			}
+			possiblePoolPaths = append(possiblePoolPaths, modernPath)
+		}
+
+		if pool.supportLegacyPaths && checksums.MD5 != "" {
+			legacyPath, err := pool.LegacyPath(basename, checksums)
+			if err != nil {
+				return "", false, err
+			}
+			possiblePoolPaths = append(possiblePoolPaths, legacyPath)
+		}
+	}
+
+	for _, path := range possiblePoolPaths {
+		fullPoolPath := filepath.Join(pool.rootPath, path)
+
+		targetInfo, err := os.Stat(fullPoolPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				// unable to stat target location?
+				return "", false, err
+			}
+			// doesn't exist, skip it
+			continue
+		}
+
+		if targetInfo.Size() != checksums.Size {
+			// oops, wrong file?
+			continue
+		}
+
+		var targetChecksums *utils.ChecksumInfo
+		targetChecksums, err = pool.ensureChecksums(path, fullPoolPath, checksumStorage)
+
+		if err != nil {
+			return "", false, err
+		}
+
+		if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
+			checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
+			// wrong file?
+			return "", false, nil
+		}
+
+		// fill back checksums
+		*checksums = *targetChecksums
+		return path, true, nil
+	}
+
+	return "", false, nil
+}
+
 // Import copies file into package pool
-func (pool *PackagePool) Import(path string, hashMD5 string) error {
+//
+// - srcPath is full path to source file as it is now
+// - basename is desired human-readable name (canonical filename)
+// - checksums are used to calculate file placement
+// - move indicates whether srcPath can be removed
+func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, checksumStorage aptly.ChecksumStorage) (string, error) {
 	pool.Lock()
 	defer pool.Unlock()
 
-	source, err := os.Open(path)
+	source, err := os.Open(srcPath)
 	if err != nil {
-		return err
+		return "", err
 	}
 	defer source.Close()
 
 	sourceInfo, err := source.Stat()
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	poolPath, err := pool.Path(path, hashMD5)
+	if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.Size != sourceInfo.Size() {
+		// need to update checksums, MD5 and SHA256 should be always defined
+		*checksums, err = utils.ChecksumsForFile(srcPath)
 		if err != nil {
-		return err
+			return "", err
+		}
 	}
 
-	targetInfo, err := os.Stat(poolPath)
+	// build target path
+	poolPath, err := pool.buildPoolPath(basename, checksums)
+	if err != nil {
+		return "", err
+	}
+
+	fullPoolPath := filepath.Join(pool.rootPath, poolPath)
+
+	targetInfo, err := os.Stat(fullPoolPath)
 	if err != nil {
 		if !os.IsNotExist(err) {
 			// unable to stat target location?
-			return err
+			return "", err
 		}
 	} else {
-		// target already exists
+		// target already exists and same size
-		if targetInfo.Size() != sourceInfo.Size() {
+		if targetInfo.Size() == sourceInfo.Size() {
-			// trying to overwrite file?
+			var targetChecksums *utils.ChecksumInfo
-			return fmt.Errorf("unable to import into pool: file %s already exists", poolPath)
+
+			targetChecksums, err = pool.ensureChecksums(poolPath, fullPoolPath, checksumStorage)
+			if err != nil {
+				return "", err
 			}
 
-		// assume that target is already there
+			*checksums = *targetChecksums
-		return nil
+			return poolPath, nil
+		}
+
+		// trying to overwrite file?
+		return "", fmt.Errorf("unable to import into pool: file %s already exists", fullPoolPath)
+	}
+
+	if pool.supportLegacyPaths {
+		// file doesn't exist at new location, check legacy location
+		var (
+			legacyTargetInfo           os.FileInfo
+			legacyPath, legacyFullPath string
+		)
+
+		legacyPath, err = pool.LegacyPath(basename, checksums)
+		if err != nil {
+			return "", err
+		}
+		legacyFullPath = filepath.Join(pool.rootPath, legacyPath)
+
+		legacyTargetInfo, err = os.Stat(legacyFullPath)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return "", err
+			}
+		} else {
+			// legacy file exists
+			if legacyTargetInfo.Size() == sourceInfo.Size() {
+				// file exists at legacy path and it's same size, consider it's already in the pool
+				var targetChecksums *utils.ChecksumInfo
+
+				targetChecksums, err = pool.ensureChecksums(legacyPath, legacyFullPath, checksumStorage)
+				if err != nil {
+					return "", err
+				}
+
+				*checksums = *targetChecksums
+				return legacyPath, nil
+			}
+
+			// size is different, import at new path
+		}
 	}
 
 	// create subdirs as necessary
-	err = os.MkdirAll(filepath.Dir(poolPath), 0777)
+	poolDir := filepath.Dir(fullPoolPath)
+	err = os.MkdirAll(poolDir, 0777)
 	if err != nil {
-		return err
+		return "", err
 	}
 
-	target, err := os.Create(poolPath)
+	// check if we can use hardlinks instead of copying/moving
+	poolDirInfo, err := os.Stat(poolDir)
 	if err != nil {
-		return err
+		return "", err
+	}
+
+	if poolDirInfo.Sys().(*syscall.Stat_t).Dev == sourceInfo.Sys().(*syscall.Stat_t).Dev {
+		// same filesystem, try to use hardlink
+		err = os.Link(srcPath, fullPoolPath)
+	} else {
+		err = os.ErrInvalid
+	}
+
+	if err != nil {
+		// different filesystems or failed hardlink, fallback to copy
+		var target *os.File
+		target, err = os.Create(fullPoolPath)
+		if err != nil {
+			return "", err
 		}
 		defer target.Close()
 
 		_, err = io.Copy(target, source)
 
-	return err
+		if err == nil {
+			err = target.Close()
+		}
+	}
+
+	if err == nil {
+		if !checksums.Complete() {
+			// need full checksums here
+			*checksums, err = utils.ChecksumsForFile(srcPath)
+			if err != nil {
+				return "", err
+			}
+		}
+
+		err = checksumStorage.Update(poolPath, checksums)
+	}
+
+	if err == nil && move {
+		err = os.Remove(srcPath)
+	}
+
+	return poolPath, err
+}
+
+// Open returns io.ReadCloser to access the file
+func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	return os.Open(filepath.Join(pool.rootPath, path))
+}
+
+// Stat returns Unix stat(2) info
+func (pool *PackagePool) Stat(path string) (os.FileInfo, error) {
+	return os.Stat(filepath.Join(pool.rootPath, path))
+}
+
+// Link generates hardlink to destination path
+func (pool *PackagePool) Link(path, dstPath string) error {
+	return os.Link(filepath.Join(pool.rootPath, path), dstPath)
+}
+
+// Symlink generates symlink to destination path
+func (pool *PackagePool) Symlink(path, dstPath string) error {
+	return os.Symlink(filepath.Join(pool.rootPath, path), dstPath)
+}
+
+// FullPath generates full path to the file in pool
+//
+// Please use with care: it's not supposed to be used to access files
+func (pool *PackagePool) FullPath(path string) string {
+	return filepath.Join(pool.rootPath, path)
+}
+
+// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
+func (pool *PackagePool) GenerateTempPath(filename string) (string, error) {
+	random := uuid.NewRandom().String()
+
+	return filepath.Join(pool.rootPath, random[0:2], random[2:4], random[4:]+filename), nil
 }

files/package_pool_test.go

@@ -1,47 +1,51 @@
 package files
 
 import (
+	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 	"runtime"
+	"syscall"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 
 	. "gopkg.in/check.v1"
 )
 
 type PackagePoolSuite struct {
 	pool     *PackagePool
+	checksum utils.ChecksumInfo
+	debFile  string
+	cs       aptly.ChecksumStorage
 }
 
 var _ = Suite(&PackagePoolSuite{})
 
 func (s *PackagePoolSuite) SetUpTest(c *C) {
-	s.pool = NewPackagePool(c.MkDir())
+	s.pool = NewPackagePool(c.MkDir(), true)
-
+	s.checksum = utils.ChecksumInfo{
+		MD5: "0035d7822b2f8f0ec4013f270fd650c2",
+	}
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.cs = NewMockChecksumStorage()
 }
 
-func (s *PackagePoolSuite) TestRelativePath(c *C) {
+func (s *PackagePoolSuite) TestLegacyPath(c *C) {
-	path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
+	path, err := s.pool.LegacyPath("a/b/package.deb", &s.checksum)
 	c.Assert(err, IsNil)
-	c.Assert(path, Equals, "91/b1/package.deb")
+	c.Assert(path, Equals, "00/35/package.deb")
 
-	_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
+	_, err = s.pool.LegacyPath("/", &s.checksum)
 	c.Assert(err, ErrorMatches, ".*is invalid")
-	_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
+	_, err = s.pool.LegacyPath("", &s.checksum)
 	c.Assert(err, ErrorMatches, ".*is invalid")
-	_, err = s.pool.RelativePath("a/b/package.deb", "9")
+	_, err = s.pool.LegacyPath("a/b/package.deb", &utils.ChecksumInfo{MD5: "9"})
 	c.Assert(err, ErrorMatches, ".*MD5 is missing")
 }
 
-func (s *PackagePoolSuite) TestPath(c *C) {
-	path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, IsNil)
-	c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
-
-	_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, ErrorMatches, ".*is invalid")
-}
-
 func (s *PackagePoolSuite) TestFilepathList(c *C) {
 	list, err := s.pool.FilepathList(nil)
 	c.Check(err, IsNil)
@@ -88,33 +92,264 @@ func (s *PackagePoolSuite) TestRemove(c *C) {
 }
 
 func (s *PackagePoolSuite) TestImportOk(c *C) {
-	_, _File, _, _ := runtime.Caller(0)
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
-	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
-
-	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
 	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// SHA256 should be automatically calculated
+	c.Check(s.checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*mockChecksumStorage).store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
 
-	info, err := os.Stat(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	info, err := s.pool.Stat(path)
-	c.Check(err, IsNil)
+	c.Assert(err, IsNil)
 	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 1, Equals, true)
+
+	// import as different name
+	path, err = s.pool.Import(s.debFile, "some.deb", &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
+	// checksum storage is filled with new checksum
+	c.Check(s.cs.(*mockChecksumStorage).store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
 
 	// double import, should be ok
-	err = s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
 	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// clear checksum storage, and do double-import
+	delete(s.cs.(*mockChecksumStorage).store, path)
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on re-calculation of file in the pool
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// import under new name, but with checksums already filled in
+	s.checksum.SHA512 = "" // clear checksum
+	path, err = s.pool.Import(s.debFile, "other.deb", &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
+	// checksum is filled back based on re-calculation of source file
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestImportLegacy(c *C) {
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "00", "35"), 0755)
+	err := utils.CopyFile(s.debFile, filepath.Join(s.pool.rootPath, "00", "35", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	c.Assert(err, IsNil)
+
+	s.checksum.Size = 2738
+	var path string
+	path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerifyLegacy(c *C) {
+	s.checksum.Size = 2738
+	// file doesn't exist yet
+	path, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(path, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "00", "35"), 0755)
+	err = utils.CopyFile(s.debFile, filepath.Join(s.pool.rootPath, "00", "35", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	c.Assert(err, IsNil)
+
+	// check existence (and fills back checksum)
+	path, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(path, Equals, "00/35/libboost-program-options-dev_1.49.0.1_i386.deb")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+}
+
+func (s *PackagePoolSuite) TestVerify(c *C) {
+	// file doesn't exist yet
+	ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// import file
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	// check existence
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, ppath)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence with fixed path
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, but with missing checksum
+	s.checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with missing checksum info but correct path and size available
+	ck := utils.ChecksumInfo{
+		Size: s.checksum.Size,
+	}
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &ck, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on checksum storage
+	c.Check(ck.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong checksum info but correct path and size available
+	ck.SHA256 = "abc"
+	ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &ck, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with missing checksum and no info in checksum storage
+	delete(s.cs.(*mockChecksumStorage).store, path)
+	s.checksum.SHA512 = ""
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, path)
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, true)
+	// checksum is filled back based on re-calculation
+	c.Check(s.checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
+
+	// check existence, with wrong size
+	s.checksum.Size = 13455
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &s.checksum, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+
+	// check existence, with empty checksum info
+	ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
+	c.Check(ppath, Equals, "")
+	c.Check(err, IsNil)
+	c.Check(exists, Equals, false)
+}
+
+func (s *PackagePoolSuite) TestImportMove(c *C) {
+	tmpDir := c.MkDir()
+	tmpPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+
+	dst, err := os.Create(tmpPath)
+	c.Assert(err, IsNil)
+
+	src, err := os.Open(s.debFile)
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(dst, src)
+	c.Assert(err, IsNil)
+
+	c.Assert(dst.Close(), IsNil)
+	c.Assert(src.Close(), IsNil)
+
+	path, err := s.pool.Import(tmpPath, filepath.Base(tmpPath), &s.checksum, true, s.cs)
+	c.Check(err, IsNil)
+	c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	info, err := s.pool.Stat(path)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(int(info.Sys().(*syscall.Stat_t).Nlink), Equals, 1)
 }
 
 func (s *PackagePoolSuite) TestImportNotExist(c *C) {
-	err := s.pool.Import("no-such-file", "91b1a1480b90b9e269ca44d897b12575")
+	_, err := s.pool.Import("no-such-file", "a.deb", &s.checksum, false, s.cs)
 	c.Check(err, ErrorMatches, ".*no such file or directory")
 }
 
 func (s *PackagePoolSuite) TestImportOverwrite(c *C) {
-	_, _File, _, _ := runtime.Caller(0)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "c7", "6b"), 0755)
-	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "c7", "6b", "4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
 
-	os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755)
+	_, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
-	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
-
-	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
 	c.Check(err, ErrorMatches, "unable to import into pool.*")
 }
+
+func (s *PackagePoolSuite) TestStat(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	info, err := s.pool.Stat(path)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+
+	_, err = s.pool.Stat("do/es/ntexist")
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *PackagePoolSuite) TestOpen(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	f, err := s.pool.Open(path)
+	c.Assert(err, IsNil)
+	contents, err := ioutil.ReadAll(f)
+	c.Assert(err, IsNil)
+	c.Check(len(contents), Equals, 2738)
+	c.Check(f.Close(), IsNil)
+
+	_, err = s.pool.Open("do/es/ntexist")
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *PackagePoolSuite) TestLink(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	tmpDir := c.MkDir()
+	dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+	c.Check(s.pool.Link(path, dstPath), IsNil)
+
+	info, err := os.Stat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true)
+}
+
+func (s *PackagePoolSuite) TestSymlink(c *C) {
+	path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &s.checksum, false, s.cs)
+	c.Check(err, IsNil)
+
+	tmpDir := c.MkDir()
+	dstPath := filepath.Join(tmpDir, filepath.Base(s.debFile))
+	c.Check(s.pool.Symlink(path, dstPath), IsNil)
+
+	info, err := os.Stat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+	c.Check(info.Sys().(*syscall.Stat_t).Nlink > 2, Equals, true)
+
+	info, err = os.Lstat(dstPath)
+	c.Assert(err, IsNil)
+	c.Check(int(info.Sys().(*syscall.Stat_t).Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
+}
+
+func (s *PackagePoolSuite) TestGenerateRandomPath(c *C) {
+	path, err := s.pool.GenerateTempPath("a.deb")
+	c.Check(err, IsNil)
+
+	c.Check(path, Matches, ".+/[0-9a-f][0-9a-f]/[0-9a-f][0-9a-f]/[0-9a-f-]+a\\.deb")
+}

files/public.go

@@ -5,25 +5,62 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strings"
 	"syscall"
 
 	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 )
 
 // PublishedStorage abstract file system with public dirs (published repos)
 type PublishedStorage struct {
 	rootPath     string
+	linkMethod   uint
+	verifyMethod uint
 }
 
 // Check interfaces
 var (
 	_ aptly.PublishedStorage           = (*PublishedStorage)(nil)
-	_ aptly.LocalPublishedStorage = (*PublishedStorage)(nil)
+	_ aptly.FileSystemPublishedStorage = (*PublishedStorage)(nil)
+)
+
+// Constants defining the type of creating links
+const (
+	LinkMethodHardLink uint = iota
+	LinkMethodSymLink
+	LinkMethodCopy
+)
+
+// Constants defining the type of file verification for LinkMethodCopy
+const (
+	VerificationMethodChecksum uint = iota
+	VerificationMethodFileSize
 )
 
 // NewPublishedStorage creates new instance of PublishedStorage which specified root
-func NewPublishedStorage(root string) *PublishedStorage {
+func NewPublishedStorage(root string, linkMethod string, verifyMethod string) *PublishedStorage {
-	return &PublishedStorage{rootPath: filepath.Join(root, "public")}
+	// Ensure linkMethod is one of 'hardlink', 'symlink', 'copy'
+	var verifiedLinkMethod uint
+
+	if strings.EqualFold(linkMethod, "copy") {
+		verifiedLinkMethod = LinkMethodCopy
+	} else if strings.EqualFold(linkMethod, "symlink") {
+		verifiedLinkMethod = LinkMethodSymLink
+	} else {
+		verifiedLinkMethod = LinkMethodHardLink
+	}
+
+	var verifiedVerifyMethod uint
+
+	if strings.EqualFold(verifyMethod, "size") {
+		verifiedVerifyMethod = VerificationMethodFileSize
+	} else {
+		verifiedVerifyMethod = VerificationMethodChecksum
+	}
+
+	return &PublishedStorage{rootPath: root, linkMethod: verifiedLinkMethod,
+		verifyMethod: verifiedVerifyMethod}
 }
 
 // PublicPath returns root of public part
@@ -77,15 +114,12 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
 //
 // publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
 // sourcePool is instance of aptly.PackagePool
-// sourcePath is filepath to package file in package pool
+// sourcePath is a relative path to package file in package pool
 //
 // LinkFromPool returns relative path for the published file to be included in package index
-func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourcePool aptly.PackagePool,
+func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool,
-	sourcePath, sourceMD5 string, force bool) error {
+	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
-	// verify that package pool is local pool is filesystem pool
-	_ = sourcePool.(*PackagePool)
 
-	baseName := filepath.Base(sourcePath)
 	poolPath := filepath.Join(storage.rootPath, publishedDirectory)
 
 	err := os.MkdirAll(poolPath, 0777)
@@ -98,19 +132,43 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
 	if err == nil {
 		// already exists, check source file
-		srcStat, err = os.Stat(sourcePath)
+		srcStat, err = sourcePool.Stat(sourcePath)
 		if err != nil {
 			// source file doesn't exist? problem!
 			return err
 		}
 
+		if storage.linkMethod == LinkMethodCopy {
+			if storage.verifyMethod == VerificationMethodFileSize {
+				// if source and destination have the same size, no need to copy
+				if srcStat.Size() == dstStat.Size() {
+					return nil
+				}
+			} else {
+				// if source and destination have the same checksums, no need to copy
+				var dstMD5 string
+				dstMD5, err = utils.MD5ChecksumForFile(filepath.Join(poolPath, baseName))
+
+				if err != nil {
+					return err
+				}
+
+				if dstMD5 == sourceChecksums.MD5 {
+					return nil
+				}
+			}
+		} else {
 			srcSys := srcStat.Sys().(*syscall.Stat_t)
 			dstSys := dstStat.Sys().(*syscall.Stat_t)
 
-		// source and destination inodes match, no need to link
+			// if source and destination inodes match, no need to link
-		if srcSys.Ino == dstSys.Ino {
+
+			// Symlink can point to different filesystem with identical inodes
+			// so we have to check the device as well.
+			if srcSys.Ino == dstSys.Ino && srcSys.Dev == dstSys.Dev {
 				return nil
 			}
+		}
 
 		// source and destination have different inodes, if !forced, this is fatal error
 		if !force {
@@ -124,8 +182,42 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory string, sourceP
 		}
 	}
 
-	// destination doesn't exist (or forced), create link
+	// destination doesn't exist (or forced), create link or copy
-	return os.Link(sourcePath, filepath.Join(poolPath, baseName))
+	if storage.linkMethod == LinkMethodCopy {
+		var r aptly.ReadSeekerCloser
+		r, err = sourcePool.Open(sourcePath)
+		if err != nil {
+			return err
+		}
+
+		var dst *os.File
+		dst, err = os.Create(filepath.Join(poolPath, baseName))
+		if err != nil {
+			r.Close()
+			return err
+		}
+
+		_, err = io.Copy(dst, r)
+		if err != nil {
+			r.Close()
+			dst.Close()
+			return err
+		}
+
+		err = r.Close()
+		if err != nil {
+			dst.Close()
+			return err
+		}
+
+		err = dst.Close()
+	} else if storage.linkMethod == LinkMethodSymLink {
+		err = sourcePool.(aptly.LocalPackagePool).Symlink(sourcePath, filepath.Join(poolPath, baseName))
+	} else {
+		err = sourcePool.(aptly.LocalPackagePool).Link(sourcePath, filepath.Join(poolPath, baseName))
+	}
+
+	return err
 }
 
 // Filelist returns list of files under prefix

files/public_test.go

@@ -6,23 +6,49 @@ import (
 	"path/filepath"
 	"syscall"
 
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type PublishedStorageSuite struct {
 	root            string
 	storage         *PublishedStorage
+	storageSymlink  *PublishedStorage
+	storageCopy     *PublishedStorage
+	storageCopySize *PublishedStorage
+	cs              aptly.ChecksumStorage
 }
 
 var _ = Suite(&PublishedStorageSuite{})
 
 func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	s.root = c.MkDir()
-	s.storage = NewPublishedStorage(s.root)
+	s.storage = NewPublishedStorage(filepath.Join(s.root, "public"), "", "")
+	s.storageSymlink = NewPublishedStorage(filepath.Join(s.root, "public_symlink"), "symlink", "")
+	s.storageCopy = NewPublishedStorage(filepath.Join(s.root, "public_copy"), "copy", "")
+	s.storageCopySize = NewPublishedStorage(filepath.Join(s.root, "public_copysize"), "copy", "size")
+	s.cs = NewMockChecksumStorage()
+}
+
+func (s *PublishedStorageSuite) TestLinkMethodField(c *C) {
+	c.Assert(s.storage.linkMethod, Equals, LinkMethodHardLink)
+	c.Assert(s.storageSymlink.linkMethod, Equals, LinkMethodSymLink)
+	c.Assert(s.storageCopy.linkMethod, Equals, LinkMethodCopy)
+	c.Assert(s.storageCopySize.linkMethod, Equals, LinkMethodCopy)
+}
+
+func (s *PublishedStorageSuite) TestVerifyMethodField(c *C) {
+	c.Assert(s.storageCopy.verifyMethod, Equals, VerificationMethodChecksum)
+	c.Assert(s.storageCopySize.verifyMethod, Equals, VerificationMethodFileSize)
 }
 
 func (s *PublishedStorageSuite) TestPublicPath(c *C) {
 	c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
+	c.Assert(s.storageSymlink.PublicPath(), Equals, filepath.Join(s.root, "public_symlink"))
+	c.Assert(s.storageCopy.PublicPath(), Equals, filepath.Join(s.root, "public_copy"))
+	c.Assert(s.storageCopySize.PublicPath(), Equals, filepath.Join(s.root, "public_copysize"))
 }
 
 func (s *PublishedStorageSuite) TestMkDir(c *C) {
@@ -33,7 +59,7 @@ func (s *PublishedStorageSuite) TestMkDir(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *PublishedStorageSuite) TesPutFile(c *C) {
+func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
@@ -85,6 +111,7 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	c.Assert(err, IsNil)
 
 	err = s.storage.RemoveDirs("ppa/dists/", nil)
+	c.Assert(err, IsNil)
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
@@ -99,6 +126,7 @@ func (s *PublishedStorageSuite) TestRemove(c *C) {
 	c.Assert(err, IsNil)
 
 	err = s.storage.Remove("ppa/dists/squeeze/Release")
+	c.Assert(err, IsNil)
 
 	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
@@ -116,78 +144,133 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 		{ // package name regular
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			sourcePath:       "mars-invaders_1.03.deb",
 			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // lib-like filename
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/libmars-invaders_1.03.deb",
+			sourcePath:       "libmars-invaders_1.03.deb",
 			poolDirectory:    "libm/libmars-invaders",
 			expectedFilename: "pool/main/libm/libmars-invaders/libmars-invaders_1.03.deb",
 		},
 		{ // duplicate link, shouldn't panic
 			prefix:           "",
 			component:        "main",
-			sourcePath:       "pool/01/ae/mars-invaders_1.03.deb",
+			sourcePath:       "mars-invaders_1.03.deb",
 			poolDirectory:    "m/mars-invaders",
 			expectedFilename: "pool/main/m/mars-invaders/mars-invaders_1.03.deb",
 		},
 		{ // prefix & component
 			prefix:           "ppa",
 			component:        "contrib",
-			sourcePath:       "pool/01/ae/libmars-invaders_1.04.deb",
+			sourcePath:       "libmars-invaders_1.04.deb",
 			poolDirectory:    "libm/libmars-invaders",
 			expectedFilename: "pool/contrib/libm/libmars-invaders/libmars-invaders_1.04.deb",
 		},
 	}
 
-	pool := NewPackagePool(s.root)
+	pool := NewPackagePool(s.root, false)
 
 	for _, t := range tests {
-		t.sourcePath = filepath.Join(s.root, t.sourcePath)
+		tmpPath := filepath.Join(c.MkDir(), t.sourcePath)
-
+		err := ioutil.WriteFile(tmpPath, []byte("Contents"), 0644)
-		err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
 		c.Assert(err, IsNil)
 
-		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
+		sourceChecksum, err := utils.ChecksumsForFile(tmpPath)
 		c.Assert(err, IsNil)
 
-		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), pool, t.sourcePath, "", false)
+		srcPoolPath, err := pool.Import(tmpPath, t.sourcePath, &utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}, false, s.cs)
+		c.Assert(err, IsNil)
+
+		// Test using hardlinks
+		err = s.storage.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
 		c.Assert(err, IsNil)
 
 		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
 		c.Assert(err, IsNil)
 
 		info := st.Sys().(*syscall.Stat_t)
-		c.Check(int(info.Nlink), Equals, 2)
+		c.Check(int(info.Nlink), Equals, 3)
-	}
 
-	// test linking files to duplicate final name
+		// Test using symlinks
-	sourcePath := filepath.Join(s.root, "pool/02/bc/mars-invaders_1.03.deb")
+		err = s.storageSymlink.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
-	err := os.MkdirAll(filepath.Dir(sourcePath), 0755)
 		c.Assert(err, IsNil)
 
-	err = ioutil.WriteFile(sourcePath, []byte("Contents"), 0644)
+		st, err = os.Lstat(filepath.Join(s.storageSymlink.rootPath, t.prefix, t.expectedFilename))
-	c.Assert(err, IsNil)
-
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", false)
-	c.Check(err, ErrorMatches, ".*file already exists and is different")
-
-	st, err := os.Stat(sourcePath)
-	c.Assert(err, IsNil)
-
-	info := st.Sys().(*syscall.Stat_t)
-	c.Check(int(info.Nlink), Equals, 1)
-
-	// linking with force
-	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), pool, sourcePath, "", true)
-	c.Check(err, IsNil)
-
-	st, err = os.Stat(sourcePath)
 		c.Assert(err, IsNil)
 
 		info = st.Sys().(*syscall.Stat_t)
-	c.Check(int(info.Nlink), Equals, 2)
+		c.Check(int(info.Nlink), Equals, 1)
+		c.Check(int(info.Mode&syscall.S_IFMT), Equals, int(syscall.S_IFLNK))
+
+		// Test using copy with checksum verification
+		err = s.storageCopy.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		c.Assert(err, IsNil)
+
+		st, err = os.Stat(filepath.Join(s.storageCopy.rootPath, t.prefix, t.expectedFilename))
+		c.Assert(err, IsNil)
+
+		info = st.Sys().(*syscall.Stat_t)
+		c.Check(int(info.Nlink), Equals, 1)
+
+		// Test using copy with size verification
+		err = s.storageCopySize.LinkFromPool(filepath.Join(t.prefix, "pool", t.component, t.poolDirectory), t.sourcePath, pool, srcPoolPath, sourceChecksum, false)
+		c.Assert(err, IsNil)
+
+		st, err = os.Stat(filepath.Join(s.storageCopySize.rootPath, t.prefix, t.expectedFilename))
+		c.Assert(err, IsNil)
+
+		info = st.Sys().(*syscall.Stat_t)
+		c.Check(int(info.Nlink), Equals, 1)
+	}
+
+	// test linking files to duplicate final name
+	tmpPath := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
+	err := ioutil.WriteFile(tmpPath, []byte("cONTENTS"), 0644)
+	c.Assert(err, IsNil)
+
+	sourceChecksum, err := utils.ChecksumsForFile(tmpPath)
+	c.Assert(err, IsNil)
+
+	srcPoolPath, err := pool.Import(tmpPath, "mars-invaders_1.03.deb", &utils.ChecksumInfo{MD5: "02bcda7a1ce305a3b60af9d5733ac1d"}, true, s.cs)
+	c.Assert(err, IsNil)
+
+	st, err := pool.Stat(srcPoolPath)
+	c.Assert(err, IsNil)
+	nlinks := int(st.Sys().(*syscall.Stat_t).Nlink)
+
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	st, err = pool.Stat(srcPoolPath)
+	c.Assert(err, IsNil)
+	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks)
+
+	// linking with force
+	err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	c.Check(err, IsNil)
+
+	st, err = pool.Stat(srcPoolPath)
+	c.Assert(err, IsNil)
+	c.Check(int(st.Sys().(*syscall.Stat_t).Nlink), Equals, nlinks+1)
+
+	// Test using symlinks
+	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	err = s.storageSymlink.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	c.Check(err, IsNil)
+
+	// Test using copy with checksum verification
+	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, ErrorMatches, ".*file already exists and is different")
+
+	err = s.storageCopy.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, true)
+	c.Check(err, IsNil)
+
+	// Test using copy with size verification (this will NOT detect the difference)
+	err = s.storageCopySize.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, srcPoolPath, sourceChecksum, false)
+	c.Check(err, IsNil)
 }

http/compression.go

@@ -0,0 +1,95 @@
+package http
+
+import (
+	"compress/bzip2"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"strings"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	xz "github.com/smira/go-xz"
+)
+
+// List of extensions + corresponding uncompression support
+var compressionMethods = []struct {
+	extenstion     string
+	transformation func(io.Reader) (io.Reader, error)
+}{
+	{
+		extenstion:     ".bz2",
+		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
+	},
+	{
+		extenstion:     ".gz",
+		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
+	},
+	{
+		extenstion:     ".xz",
+		transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
+	},
+	{
+		extenstion:     "",
+		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
+	},
+}
+
+// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
+// it finds existing file.
+func DownloadTryCompression(downloader aptly.Downloader, baseURL *url.URL, path string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
+	var err error
+
+	for _, method := range compressionMethods {
+		var file *os.File
+
+		tryPath := path + method.extenstion
+		foundChecksum := false
+
+		bestSuffix := ""
+
+		for suffix := range expectedChecksums {
+			if strings.HasSuffix(tryPath, suffix) {
+				foundChecksum = true
+				if len(suffix) > len(bestSuffix) {
+					bestSuffix = suffix
+				}
+			}
+		}
+
+		tryURL := baseURL.ResolveReference(&url.URL{Path: tryPath})
+
+		if foundChecksum {
+			expected := expectedChecksums[bestSuffix]
+			file, err = DownloadTempWithChecksum(downloader, tryURL.String(), &expected, ignoreMismatch, maxTries)
+		} else {
+			if !ignoreMismatch {
+				continue
+			}
+
+			file, err = DownloadTemp(downloader, tryURL.String())
+		}
+
+		if err != nil {
+			if err1, ok := err.(*Error); ok && (err1.Code == 404 || err1.Code == 403) {
+				continue
+			}
+			return nil, nil, err
+		}
+
+		var uncompressed io.Reader
+		uncompressed, err = method.transformation(file)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		return uncompressed, file, err
+	}
+
+	if err == nil {
+		err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path}))
+	}
+	return nil, nil, err
+}

http/compression_test.go

@@ -0,0 +1,146 @@
+package http
+
+import (
+	"errors"
+	"io"
+	"net/url"
+
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type CompressionSuite struct {
+	baseURL *url.URL
+}
+
+var _ = Suite(&CompressionSuite{})
+
+const (
+	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
+	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
+	xzData   = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
+	rawData  = "test"
+)
+
+func (s *CompressionSuite) SetUpTest(c *C) {
+	s.baseURL, _ = url.Parse("http://example.com/")
+}
+
+func (s *CompressionSuite) TestDownloadTryCompression(c *C) {
+	var buf []byte
+
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: int64(len(bzipData))},
+		"file.gz":  {Size: int64(len(gzipData))},
+		"file.xz":  {Size: int64(len(xzData))},
+		"file":     {Size: int64(len(rawData))},
+	}
+
+	// bzip2 only available
+	buf = make([]byte, 4)
+	d := NewFakeDownloader()
+	d.ExpectResponse("http://example.com/file.bz2", bzipData)
+	r, file, err := DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 not available, but gz is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.gz", gzipData)
+	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 & gzip not available, but xz is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.xz", xzData)
+	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2, gzip & xz not available, but raw is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file", rawData)
+	r, file, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// gzip available, but broken
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file.gz", "x")
+	_, _, err = DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "unexpected EOF")
+	c.Assert(d.Empty(), Equals, true)
+}
+
+func (s *CompressionSuite) TestDownloadTryCompressionLongestSuffix(c *C) {
+	var buf []byte
+
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2":          {Size: 1},
+		"subdir/file.bz2":   {Size: int64(len(bzipData))},
+		"otherdir/file.bz2": {Size: 1},
+	}
+
+	// longest suffix should be picked up
+	buf = make([]byte, 4)
+	d := NewFakeDownloader()
+	d.ExpectResponse("http://example.com/subdir/file.bz2", bzipData)
+	r, file, err := DownloadTryCompression(d, s.baseURL, "subdir/file", expectedChecksums, false, 1)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+}
+
+func (s *CompressionSuite) TestDownloadTryCompressionErrors(c *C) {
+	d := NewFakeDownloader()
+	_, _, err := DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "unexpected request.*")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file", errors.New("403"))
+	_, _, err = DownloadTryCompression(d, s.baseURL, "file", nil, true, 1)
+	c.Assert(err, ErrorMatches, "403")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
+	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
+	d.ExpectResponse("http://example.com/file", rawData)
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": {Size: 7},
+		"file.gz":  {Size: 7},
+		"file.xz":  {Size: 7},
+		"file":     {Size: 7},
+	}
+	_, _, err = DownloadTryCompression(d, s.baseURL, "file", expectedChecksums, false, 1)
+	c.Assert(err, ErrorMatches, "checksums don't match.*")
+}

http/download.go

@@ -1,11 +1,8 @@
 package http
 
 import (
-	"compress/bzip2"
-	"compress/gzip"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -13,23 +10,12 @@ import (
 	"time"
 
 	"github.com/mxk/go-flowrate/flowrate"
+	"github.com/pkg/errors"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/go-ftp-protocol/protocol"
-	"github.com/smira/go-xz"
 )
 
-// Error is download error connected to HTTP code
-type Error struct {
-	Code int
-	URL  string
-}
-
-// Error
-func (e *Error) Error() string {
-	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
-}
-
 // Check interface
 var (
 	_ aptly.Downloader = (*downloaderImpl)(nil)
@@ -37,30 +23,14 @@ var (
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue     chan *downloadTask
-	stop      chan struct{}
-	stopped   chan struct{}
-	pause     chan struct{}
-	unpause   chan struct{}
 	progress  aptly.Progress
 	aggWriter io.Writer
-	threads   int
 	client    *http.Client
 }
 
-// downloadTask represents single item in queue
-type downloadTask struct {
-	url            string
-	destination    string
-	result         chan<- error
-	expected       utils.ChecksumInfo
-	ignoreMismatch bool
-	triesLeft      int
-}
-
 // NewDownloader creates new instance of Downloader which specified number
 // of threads and download limit in bytes/sec
-func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
+func NewDownloader(downLimit int64, progress aptly.Progress) aptly.Downloader {
 	transport := http.Transport{}
 	transport.Proxy = http.DefaultTransport.(*http.Transport).Proxy
 	transport.ResponseHeaderTimeout = 30 * time.Second
@@ -71,12 +41,6 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 	transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
 
 	downloader := &downloaderImpl{
-		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan struct{}, threads),
-		stopped:  make(chan struct{}, threads),
-		pause:    make(chan struct{}),
-		unpause:  make(chan struct{}),
-		threads:  threads,
 		progress: progress,
 		client: &http.Client{
 			Transport: &transport,
@@ -89,70 +53,28 @@ func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.
 		downloader.aggWriter = progress
 	}
 
-	for i := 0; i < downloader.threads; i++ {
-		go downloader.process()
-	}
-
 	return downloader
 }
 
-// Shutdown stops downloader after current tasks are finished,
-// but doesn't process rest of queue
-func (downloader *downloaderImpl) Shutdown() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- struct{}{}
-	}
-
-	for i := 0; i < downloader.threads; i++ {
-		<-downloader.stopped
-	}
-}
-
-// Abort stops downloader but doesn't wait for downloader to stop
-func (downloader *downloaderImpl) Abort() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.stop <- struct{}{}
-	}
-}
-
-// Pause pauses task processing
-func (downloader *downloaderImpl) Pause() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.pause <- struct{}{}
-	}
-}
-
-// Resume resumes task processing
-func (downloader *downloaderImpl) Resume() {
-	for i := 0; i < downloader.threads; i++ {
-		downloader.unpause <- struct{}{}
-	}
-}
-
 // GetProgress returns Progress object
 func (downloader *downloaderImpl) GetProgress() aptly.Progress {
 	return downloader.progress
 }
 
 // Download starts new download task
-func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
+func (downloader *downloaderImpl) Download(url string, destination string) error {
-	downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false, 1)
+	return downloader.DownloadWithChecksum(url, destination, nil, false, 1)
 }
 
 // DownloadWithChecksum starts new download task with checksum verification
-func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
+func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string,
-	expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
+	expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
-	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch, triesLeft: maxTries}
-}
 
-// handleTask processes single download task
+	downloader.progress.Printf("Downloading %s...\n", url)
-func (downloader *downloaderImpl) handleTask(task *downloadTask) {
-	downloader.progress.Printf("Downloading %s...\n", task.url)
 
-	req, err := http.NewRequest("GET", task.url, nil)
+	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return errors.Wrap(err, url)
-		return
 	}
 	req.Close = true
 
@@ -163,12 +85,11 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	}
 
 	var temppath string
-	for task.triesLeft > 0 {
+	for maxTries > 0 {
-
+		temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch)
-		temppath, err = downloader.downloadTask(req, task)
 
 		if err != nil {
-			task.triesLeft--
+			maxTries--
 		} else {
 			// successful download
 			break
@@ -177,50 +98,48 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	// still an error after retrying, giving up
 	if err != nil {
-		task.result <- err
+		return err
-		return
 	}
 
-	err = os.Rename(temppath, task.destination)
+	err = os.Rename(temppath, destination)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- fmt.Errorf("%s: %s", task.url, err)
+		return errors.Wrap(err, url)
-		return
 	}
 
-	task.result <- nil
+	return nil
 }
 
-func (downloader *downloaderImpl) downloadTask(req *http.Request, task *downloadTask) (string, error) {
+func (downloader *downloaderImpl) download(req *http.Request, url, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) (string, error) {
 	resp, err := downloader.client.Do(req)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 	if resp.Body != nil {
 		defer resp.Body.Close()
 	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return "", &Error{Code: resp.StatusCode, URL: task.url}
+		return "", &Error{Code: resp.StatusCode, URL: url}
 	}
 
-	err = os.MkdirAll(filepath.Dir(task.destination), 0777)
+	err = os.MkdirAll(filepath.Dir(destination), 0777)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 
-	temppath := task.destination + ".down"
+	temppath := destination + ".down"
 
 	outfile, err := os.Create(temppath)
 	if err != nil {
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 	defer outfile.Close()
 
 	checksummer := utils.NewChecksumWriter()
 	writers := []io.Writer{outfile, downloader.aggWriter}
 
-	if task.expected.Size != -1 {
+	if expected != nil {
 		writers = append(writers, checksummer)
 	}
 
@@ -229,161 +148,36 @@ func (downloader *downloaderImpl) downloadTask(req *http.Request, task *download
 	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
-		return "", fmt.Errorf("%s: %s", task.url, err)
+		return "", errors.Wrap(err, url)
 	}
 
-	if task.expected.Size != -1 {
+	if expected != nil {
 		actual := checksummer.Sum()
 
-		if actual.Size != task.expected.Size {
+		if actual.Size != expected.Size {
-			err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
+			err = fmt.Errorf("%s: size check mismatch %d != %d", url, actual.Size, expected.Size)
-		} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
+		} else if expected.MD5 != "" && actual.MD5 != expected.MD5 {
-			err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
+			err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", url, actual.MD5, expected.MD5)
-		} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
+		} else if expected.SHA1 != "" && actual.SHA1 != expected.SHA1 {
-			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
+			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", url, actual.SHA1, expected.SHA1)
-		} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
+		} else if expected.SHA256 != "" && actual.SHA256 != expected.SHA256 {
-			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
+			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", url, actual.SHA256, expected.SHA256)
-		} else if task.expected.SHA512 != "" && actual.SHA512 != task.expected.SHA512 {
+		} else if expected.SHA512 != "" && actual.SHA512 != expected.SHA512 {
-			err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", task.url, actual.SHA512, task.expected.SHA512)
+			err = fmt.Errorf("%s: sha512 hash mismatch %#v != %#v", url, actual.SHA512, expected.SHA512)
 		}
 
 		if err != nil {
-			if task.ignoreMismatch {
+			if ignoreMismatch {
 				downloader.progress.Printf("WARNING: %s\n", err.Error())
 			} else {
 				os.Remove(temppath)
 				return "", err
 			}
+		} else {
+			// update checksums if they match, so that they contain exactly expected set
+			*expected = actual
 		}
 	}
 
 	return temppath, nil
 }
-
-// process implements download thread in goroutine
-func (downloader *downloaderImpl) process() {
-	for {
-		select {
-		case <-downloader.stop:
-			downloader.stopped <- struct{}{}
-			return
-		case <-downloader.pause:
-			<-downloader.unpause
-		case task := <-downloader.queue:
-			downloader.handleTask(task)
-		}
-	}
-}
-
-// DownloadTemp starts new download to temporary file and returns File
-//
-// Temporary file would be already removed, so no need to cleanup
-func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
-	return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false, 1)
-}
-
-// DownloadTempWithChecksum is a DownloadTemp with checksum verification
-//
-// Temporary file would be already removed, so no need to cleanup
-func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
-	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
-	if err != nil {
-		return nil, err
-	}
-	defer os.RemoveAll(tempdir)
-
-	tempfile := filepath.Join(tempdir, "buffer")
-
-	if expected.Size != -1 && downloader.GetProgress() != nil {
-		downloader.GetProgress().InitBar(expected.Size, true)
-		defer downloader.GetProgress().ShutdownBar()
-	}
-
-	ch := make(chan error, 1)
-	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch, maxTries)
-
-	err = <-ch
-
-	if err != nil {
-		return nil, err
-	}
-
-	file, err := os.Open(tempfile)
-	if err != nil {
-		return nil, err
-	}
-
-	return file, nil
-}
-
-// List of extensions + corresponding uncompression support
-var compressionMethods = []struct {
-	extenstion     string
-	transformation func(io.Reader) (io.Reader, error)
-}{
-	{
-		extenstion:     ".bz2",
-		transformation: func(r io.Reader) (io.Reader, error) { return bzip2.NewReader(r), nil },
-	},
-	{
-		extenstion:     ".gz",
-		transformation: func(r io.Reader) (io.Reader, error) { return gzip.NewReader(r) },
-	},
-	{
-		extenstion:     ".xz",
-		transformation: func(r io.Reader) (io.Reader, error) { return xz.NewReader(r) },
-	},
-	{
-		extenstion:     "",
-		transformation: func(r io.Reader) (io.Reader, error) { return r, nil },
-	},
-}
-
-// DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
-// it finds existing file.
-func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (io.Reader, *os.File, error) {
-	var err error
-
-	for _, method := range compressionMethods {
-		var file *os.File
-
-		tryURL := url + method.extenstion
-		foundChecksum := false
-
-		for suffix, expected := range expectedChecksums {
-			if strings.HasSuffix(tryURL, suffix) {
-				file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch, maxTries)
-				foundChecksum = true
-				break
-			}
-		}
-
-		if !foundChecksum {
-			if !ignoreMismatch {
-				continue
-			}
-
-			file, err = DownloadTemp(downloader, tryURL)
-		}
-
-		if err != nil {
-			if err1, ok := err.(*Error); ok && (err1.Code == 404 || err1.Code == 403) {
-				continue
-			}
-			return nil, nil, err
-		}
-
-		var uncompressed io.Reader
-		uncompressed, err = method.transformation(file)
-		if err != nil {
-			return nil, nil, err
-		}
-
-		return uncompressed, file, err
-	}
-
-	if err == nil {
-		err = fmt.Errorf("no candidates for %s found", url)
-	}
-	return nil, nil, err
-}

http/download_test.go

@@ -1,15 +1,11 @@
 package http
 
 import (
-	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"runtime"
-	"time"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/console"
@@ -18,17 +14,16 @@ import (
 	. "gopkg.in/check.v1"
 )
 
-type DownloaderSuite struct {
+type DownloaderSuiteBase struct {
 	tempfile *os.File
 	l        net.Listener
 	url      string
-	ch       chan bool
+	ch       chan struct{}
 	progress aptly.Progress
+	d        aptly.Downloader
 }
 
-var _ = Suite(&DownloaderSuite{})
+func (s *DownloaderSuiteBase) SetUpTest(c *C) {
-
-func (s *DownloaderSuite) SetUpTest(c *C) {
 	s.tempfile, _ = ioutil.TempFile(os.TempDir(), "aptly-test")
 	s.l, _ = net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
 	s.url = fmt.Sprintf("http://localhost:%d", s.l.Addr().(*net.TCPAddr).Port)
@@ -38,18 +33,20 @@ func (s *DownloaderSuite) SetUpTest(c *C) {
 		fmt.Fprintf(w, "Hello, %s", r.URL.Path)
 	})
 
-	s.ch = make(chan bool)
+	s.ch = make(chan struct{})
 
 	go func() {
 		http.Serve(s.l, mux)
-		s.ch <- true
+		close(s.ch)
 	}()
 
 	s.progress = console.NewProgress()
 	s.progress.Start()
+
+	s.d = NewDownloader(0, s.progress)
 }
 
-func (s *DownloaderSuite) TearDownTest(c *C) {
+func (s *DownloaderSuiteBase) TearDownTest(c *C) {
 	s.progress.Shutdown()
 
 	s.l.Close()
@@ -59,249 +56,67 @@ func (s *DownloaderSuite) TearDownTest(c *C) {
 	s.tempfile.Close()
 }
 
-func (s *DownloaderSuite) TestStartupShutdown(c *C) {
+type DownloaderSuite struct {
-	goroutines := runtime.NumGoroutine()
+	DownloaderSuiteBase
-
-	d := NewDownloader(10, 100, s.progress)
-	d.Shutdown()
-
-	// wait for goroutines to shutdown
-	time.Sleep(100 * time.Millisecond)
-
-	if runtime.NumGoroutine()-goroutines > 1 {
-		c.Errorf("Number of goroutines %d, expected %d", runtime.NumGoroutine(), goroutines)
-	}
 }
 
-func (s *DownloaderSuite) TestPauseResume(c *C) {
+var _ = Suite(&DownloaderSuite{})
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
 
-	d.Pause()
+func (s *DownloaderSuite) SetUpTest(c *C) {
-	d.Resume()
+	s.DownloaderSuiteBase.SetUpTest(c)
+}
+
+func (s *DownloaderSuite) TearDownTest(c *C) {
+	s.DownloaderSuiteBase.TearDownTest(c)
 }
 
 func (s *DownloaderSuite) TestDownloadOK(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.url+"/test", s.tempfile.Name()), IsNil)
-	defer d.Shutdown()
-	ch := make(chan error)
-
-	d.Download(s.url+"/test", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, IsNil)
 }
 
 func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{}, false, 1),
-	defer d.Shutdown()
+		ErrorMatches, ".*size check mismatch 12 != 0")
-	ch := make(chan error)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1),
-	res := <-ch
+		ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
-	c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1),
-	res = <-ch
+		IsNil)
-	c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1),
-	res = <-ch
+		IsNil)
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1),
-	res = <-ch
+		ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false, 1)
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-	res = <-ch
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1),
-	c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
+		IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false, 1)
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1),
-	res = <-ch
+		ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
-	c.Assert(res, IsNil)
 
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	checksums := utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false, 1)
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}
-	res = <-ch
+	c.Assert(s.d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), &checksums, false, 1),
-	c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
+		IsNil)
-
+	// download backfills missing checksums
-	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+	c.Check(checksums.SHA512, Equals, "bac18bf4e564856369acc2ed57300fecba3a2c1af5ae8304021e4252488678feb18118466382ee4e1210fe1f065080210e453a80cfb37ccb8752af3269df160e")
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
-	res = <-ch
-	c.Assert(res, IsNil)
 }
 
 func (s *DownloaderSuite) TestDownload404(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.url+"/doesntexist", s.tempfile.Name()),
-	defer d.Shutdown()
+		ErrorMatches, "HTTP code 404.*")
-	ch := make(chan error)
-
-	d.Download(s.url+"/doesntexist", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, "HTTP code 404.*")
 }
 
 func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download("http://nosuch.localhost/", s.tempfile.Name()),
-	defer d.Shutdown()
+		ErrorMatches, ".*no such host")
-	ch := make(chan error)
-
-	d.Download("http://nosuch.localhost/", s.tempfile.Name(), ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, ".*no such host")
 }
 
 func (s *DownloaderSuite) TestDownloadFileError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
+	c.Assert(s.d.Download(s.url+"/test", "/"),
-	defer d.Shutdown()
+		ErrorMatches, ".*permission denied")
-	ch := make(chan error)
-
-	d.Download(s.url+"/test", "/", ch)
-	res := <-ch
-	c.Assert(res, ErrorMatches, ".*permission denied")
-}
-
-func (s *DownloaderSuite) TestDownloadTemp(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTemp(d, s.url+"/test")
-	c.Assert(err, IsNil)
-	defer f.Close()
-
-	buf := make([]byte, 1)
-
-	f.Read(buf)
-	c.Assert(buf, DeepEquals, []byte("H"))
-
-	_, err = os.Stat(f.Name())
-	c.Assert(os.IsNotExist(err), Equals, true)
-}
-
-func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
-		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
-	defer f.Close()
-	c.Assert(err, IsNil)
-
-	_, err = DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 13}, false, 1)
-	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
-}
-
-func (s *DownloaderSuite) TestDownloadTempError(c *C) {
-	d := NewDownloader(2, 0, s.progress)
-	defer d.Shutdown()
-
-	f, err := DownloadTemp(d, s.url+"/doesntexist")
-	c.Assert(err, NotNil)
-	c.Assert(f, IsNil)
-	c.Assert(err, ErrorMatches, "HTTP code 404.*")
-}
-
-const (
-	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
-	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
-	xzData   = "\xfd\x37\x7a\x58\x5a\x00\x00\x04\xe6\xd6\xb4\x46\x02\x00\x21\x01\x16\x00\x00\x00\x74\x2f\xe5\xa3\x01\x00\x04\x74\x65\x73\x74\x0a\x00\x00\x00\x00\x9d\xed\x31\x1d\x0f\x9f\xd7\xe6\x00\x01\x1d\x05\xb8\x2d\x80\xaf\x1f\xb6\xf3\x7d\x01\x00\x00\x00\x00\x04\x59\x5a"
-	rawData  = "test"
-)
-
-func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
-	var buf []byte
-
-	expectedChecksums := map[string]utils.ChecksumInfo{
-		"file.bz2": {Size: int64(len(bzipData))},
-		"file.gz":  {Size: int64(len(gzipData))},
-		"file.xz":  {Size: int64(len(xzData))},
-		"file":     {Size: int64(len(rawData))},
-	}
-
-	// bzip2 only available
-	buf = make([]byte, 4)
-	d := NewFakeDownloader()
-	d.ExpectResponse("http://example.com/file.bz2", bzipData)
-	r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2 not available, but gz is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.gz", gzipData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2 & gzip not available, but xz is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.xz", xzData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// bzip2, gzip & xz not available, but raw is
-	buf = make([]byte, 4)
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file", rawData)
-	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, IsNil)
-	defer file.Close()
-	io.ReadFull(r, buf)
-	c.Assert(string(buf), Equals, rawData)
-	c.Assert(d.Empty(), Equals, true)
-
-	// gzip available, but broken
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file.gz", "x")
-	_, file, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "unexpected EOF")
-	c.Assert(d.Empty(), Equals, true)
-}
-
-func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
-	d := NewFakeDownloader()
-	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "unexpected request.*")
-
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file", errors.New("403"))
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, true, 1)
-	c.Assert(err, ErrorMatches, "403")
-
-	d = NewFakeDownloader()
-	d.ExpectError("http://example.com/file.bz2", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.gz", &Error{Code: 404})
-	d.ExpectError("http://example.com/file.xz", &Error{Code: 404})
-	d.ExpectResponse("http://example.com/file", rawData)
-	expectedChecksums := map[string]utils.ChecksumInfo{
-		"file.bz2": {Size: 7},
-		"file.gz":  {Size: 7},
-		"file.xz":  {Size: 7},
-		"file":     {Size: 7},
-	}
-	_, _, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false, 1)
-	c.Assert(err, ErrorMatches, "checksums don't match.*")
 }

http/fake.go

@@ -60,7 +60,7 @@ func (f *FakeDownloader) Empty() bool {
 }
 
 // DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
-func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool, maxTries int) {
+func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error {
 	var expectation expectedRequest
 	if len(f.expected) > 0 && f.expected[0].URL == url {
 		expectation, f.expected = f.expected[0], f.expected[1:]
@@ -68,25 +68,21 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 		expectation = f.anyExpected[url]
 		delete(f.anyExpected, url)
 	} else {
-		result <- fmt.Errorf("unexpected request for %s", url)
+		return fmt.Errorf("unexpected request for %s", url)
-		return
 	}
 
 	if expectation.Err != nil {
-		result <- expectation.Err
+		return expectation.Err
-		return
 	}
 
 	err := os.MkdirAll(filepath.Dir(filename), 0755)
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 
 	outfile, err := os.Create(filename)
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 	defer outfile.Close()
 
@@ -95,45 +91,26 @@ func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, resul
 
 	_, err = w.Write([]byte(expectation.Response))
 	if err != nil {
-		result <- err
+		return err
-		return
 	}
 
-	if expected.Size != -1 {
+	if expected != nil {
 		if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
 			expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
 			if ignoreMismatch {
 				fmt.Printf("WARNING: checksums don't match: %#v != %#v for %s\n", expected, cks.Sum(), url)
 			} else {
-				result <- fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
+				return fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
-				return
 			}
 		}
 	}
 
-	result <- nil
+	return nil
-	return
 }
 
 // Download performs fake download by matching against first expectation in the queue
-func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
+func (f *FakeDownloader) Download(url string, filename string) error {
-	f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false, 1)
+	return f.DownloadWithChecksum(url, filename, nil, false, 1)
-}
-
-// Shutdown does nothing
-func (f *FakeDownloader) Shutdown() {
-}
-
-// Abort does nothing
-func (f *FakeDownloader) Abort() {
-}
-
-// Pause does nothing
-func (f *FakeDownloader) Pause() {
-}
-
-// Resume does nothing
-func (f *FakeDownloader) Resume() {
 }
 
 // GetProgress returns Progress object

http/http.go

@@ -1,2 +1,17 @@
 // Package http provides all HTTP (and FTP)-related operations
 package http
+
+import (
+	"fmt"
+)
+
+// Error is download error connected to HTTP code
+type Error struct {
+	Code int
+	URL  string
+}
+
+// Error
+func (e *Error) Error() string {
+	return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL)
+}

http/temp.go

@@ -0,0 +1,47 @@
+package http
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+)
+
+// DownloadTemp starts new download to temporary file and returns File
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
+	return DownloadTempWithChecksum(downloader, url, nil, false, 1)
+}
+
+// DownloadTempWithChecksum is a DownloadTemp with checksum verification
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) (*os.File, error) {
+	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(tempdir)
+
+	tempfile := filepath.Join(tempdir, "buffer")
+
+	if expected != nil && downloader.GetProgress() != nil {
+		downloader.GetProgress().InitBar(expected.Size, true)
+		defer downloader.GetProgress().ShutdownBar()
+	}
+
+	err = downloader.DownloadWithChecksum(url, tempfile, expected, ignoreMismatch, maxTries)
+	if err != nil {
+		return nil, err
+	}
+
+	file, err := os.Open(tempfile)
+	if err != nil {
+		return nil, err
+	}
+
+	return file, nil
+}

http/temp_test.go

@@ -0,0 +1,55 @@
+package http
+
+import (
+	"os"
+
+	"github.com/smira/aptly/utils"
+
+	. "gopkg.in/check.v1"
+)
+
+type TempSuite struct {
+	DownloaderSuiteBase
+}
+
+var _ = Suite(&TempSuite{})
+
+func (s *TempSuite) SetUpTest(c *C) {
+	s.DownloaderSuiteBase.SetUpTest(c)
+}
+
+func (s *TempSuite) TearDownTest(c *C) {
+	s.DownloaderSuiteBase.TearDownTest(c)
+}
+
+func (s *TempSuite) TestDownloadTemp(c *C) {
+	f, err := DownloadTemp(s.d, s.url+"/test")
+	c.Assert(err, IsNil)
+	defer f.Close()
+
+	buf := make([]byte, 1)
+
+	f.Read(buf)
+	c.Assert(buf, DeepEquals, []byte("H"))
+
+	_, err = os.Stat(f.Name())
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *TempSuite) TestDownloadTempWithChecksum(c *C) {
+	f, err := DownloadTempWithChecksum(s.d, s.url+"/test", &utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false, 1)
+	c.Assert(err, IsNil)
+
+	c.Assert(f.Close(), IsNil)
+
+	_, err = DownloadTempWithChecksum(s.d, s.url+"/test", &utils.ChecksumInfo{Size: 13}, false, 1)
+	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
+}
+
+func (s *TempSuite) TestDownloadTempError(c *C) {
+	f, err := DownloadTemp(s.d, s.url+"/doesntexist")
+	c.Assert(err, NotNil)
+	c.Assert(f, IsNil)
+	c.Assert(err, ErrorMatches, "HTTP code 404.*")
+}

linter.json

@@ -1,4 +1,24 @@
 {
     "DisableAll": true,
-    "Enable": ["vet", "golint", "gofmt", "deadcode", "goimports", "misspell"]
+    "Enable": [
+        "vet",
+        "golint",
+        "gofmt",
+        "deadcode",
+        "goimports",
+        "misspell",
+        "gosimple",
+        "ineffassign",
+        "staticcheck",
+        "varcheck",
+        "structcheck",
+        "aligncheck",
+        "vetshadow",
+        "goconst",
+        "interfacer"
+    ],
+    "Deadline": "20m",
+    "Vendor": true,
+    "VendoredLinters": true,
+    "Concurrency": 1
 }

lock.json

@@ -1,292 +0,0 @@
-{
-    "memo": "4ce49ad4105227467bccc48eec5832d84749da3dee8dc8f6574cd32fec3970c1",
-    "projects": [
-        {
-            "name": "github.com/AlekSi/pointer",
-            "version": "v1.0.0",
-            "revision": "08a25bac605b3fcb6cc27f3917b2c2c87451963d",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/DisposaBoy/JsonConfigReader",
-            "branch": "master",
-            "revision": "33a99fdf1d5ee1f79b5077e9c06f955ad356d5f4",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/awalterschulze/gographviz",
-            "version": "v1.0",
-            "revision": "761fd5fbb34e4c2c138c280395b65b48e4ff5a53",
-            "packages": [
-                ".",
-                "ast",
-                "parser",
-                "scanner",
-                "token"
-            ]
-        },
-        {
-            "name": "github.com/aws/aws-sdk-go",
-            "version": "v1.8.0",
-            "revision": "2db5849d2939d93075d911138309a83235032bea",
-            "packages": [
-                "aws",
-                "aws/awserr",
-                "aws/awsutil",
-                "aws/client",
-                "aws/client/metadata",
-                "aws/corehandlers",
-                "aws/credentials",
-                "aws/credentials/ec2rolecreds",
-                "aws/credentials/endpointcreds",
-                "aws/credentials/stscreds",
-                "aws/defaults",
-                "aws/ec2metadata",
-                "aws/endpoints",
-                "aws/request",
-                "aws/session",
-                "aws/signer/v4",
-                "private/protocol",
-                "private/protocol/query",
-                "private/protocol/query/queryutil",
-                "private/protocol/rest",
-                "private/protocol/restxml",
-                "private/protocol/xml/xmlutil",
-                "service/s3",
-                "service/sts"
-            ]
-        },
-        {
-            "name": "github.com/cheggaaa/pb",
-            "version": "v1.0.10",
-            "revision": "cdf719fac0dd208251aa828e687c2d5802053b51",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/gin-gonic/gin",
-            "revision": "b1758d3bfa09e61ddbc1c9a627e936eec6a170de",
-            "packages": [
-                ".",
-                "binding",
-                "render"
-            ]
-        },
-        {
-            "name": "github.com/go-ini/ini",
-            "version": "v1.26.0",
-            "revision": "1730955e3146956d6a087861380f9b4667ed5071",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/golang/snappy",
-            "branch": "master",
-            "revision": "553a641470496b2327abcac10b36396bd98e45c9",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/h2non/filetype",
-            "branch": "master",
-            "revision": "0df83c38d14ff5f653d419d480eaac286ccbc823",
-            "packages": [
-                "matchers"
-            ]
-        },
-        {
-            "name": "github.com/jlaffaye/ftp",
-            "branch": "master",
-            "revision": "7b85eb4638a2c0473acefcfb929a98f879c15c86",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/jmespath/go-jmespath",
-            "version": "0.2.2",
-            "revision": "3433f3ea46d9f8019119e7dd41274e112a2359a9",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/julienschmidt/httprouter",
-            "version": "v1.1",
-            "revision": "8c199fb6259ffc1af525cc3ad52ee60ba8359669",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mattn/go-runewidth",
-            "version": "v0.0.2",
-            "revision": "9e777a8366cce605130a531d2cd6363d07ad7317",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mattn/go-shellwords",
-            "version": "v1.0.2",
-            "revision": "005a0944d84452842197c2108bd9168ced206f78",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mkrautz/goar",
-            "branch": "master",
-            "revision": "282caa8bd9daba480b51f1d5a988714913b97aad",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/mxk/go-flowrate",
-            "branch": "master",
-            "revision": "cca7078d478f8520f85629ad7c68962d31ed7682",
-            "packages": [
-                "flowrate"
-            ]
-        },
-        {
-            "name": "github.com/ncw/swift",
-            "branch": "master",
-            "revision": "8e9b10220613abdbc2896808ee6b43e411a4fa6c",
-            "packages": [
-                ".",
-                "swifttest"
-            ]
-        },
-        {
-            "name": "github.com/smira/commander",
-            "branch": "master",
-            "revision": "f408b00e68d5d6e21b9f18bd310978dafc604e47",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/flag",
-            "branch": "master",
-            "revision": "357ed3e599ffcbd4aeaa828e1d10da2df3ea5107",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/go-aws-auth",
-            "branch": "master",
-            "revision": "0070896e9d7f4f9f2d558532b2d896ce2239992a",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/go-ftp-protocol",
-            "branch": "master",
-            "revision": "066b75c2b70dca7ae10b1b88b47534a3c31ccfaa",
-            "packages": [
-                "protocol"
-            ]
-        },
-        {
-            "name": "github.com/smira/go-uuid",
-            "branch": "master",
-            "revision": "ed3ca8a15a931b141440a7e98e4f716eec255f7d",
-            "packages": [
-                "uuid"
-            ]
-        },
-        {
-            "name": "github.com/smira/go-xz",
-            "branch": "master",
-            "revision": "0c531f070014e218b21f3cfca801cc992d52726d",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/smira/lzma",
-            "branch": "master",
-            "revision": "7f0af6269940baa2c938fabe73e0d7ba41205683",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "github.com/syndtr/goleveldb",
-            "branch": "master",
-            "revision": "3c5717caf1475fd25964109a0fc640bd150fce43",
-            "packages": [
-                "leveldb",
-                "leveldb/cache",
-                "leveldb/comparer",
-                "leveldb/errors",
-                "leveldb/filter",
-                "leveldb/iterator",
-                "leveldb/journal",
-                "leveldb/memdb",
-                "leveldb/opt",
-                "leveldb/storage",
-                "leveldb/table",
-                "leveldb/util"
-            ]
-        },
-        {
-            "name": "github.com/ugorji/go",
-            "revision": "71c2886f5a673a35f909803f38ece5810165097b",
-            "packages": [
-                "codec"
-            ]
-        },
-        {
-            "name": "github.com/wsxiaoys/terminal",
-            "branch": "master",
-            "revision": "0940f3fc43a0ed42d04916b1c04578462c650b09",
-            "packages": [
-                "color"
-            ]
-        },
-        {
-            "name": "golang.org/x/crypto",
-            "branch": "master",
-            "revision": "459e26527287adbc2adcc5d0d49abff9a5f315a7",
-            "packages": [
-                "ssh/terminal"
-            ]
-        },
-        {
-            "name": "golang.org/x/sys",
-            "branch": "master",
-            "revision": "99f16d856c9836c42d24e7ab64ea72916925fa97",
-            "packages": [
-                "unix"
-            ]
-        },
-        {
-            "name": "gopkg.in/check.v1",
-            "branch": "v1",
-            "revision": "20d25e2804050c1cd24a7eea1e7a6447dd0e74ec",
-            "packages": [
-                "."
-            ]
-        },
-        {
-            "name": "gopkg.in/h2non/filetype.v1",
-            "version": "v1.0.1",
-            "revision": "3093b8ebec6efb56ac813238b8beab4ed4eaac6a",
-            "packages": [
-                "types"
-            ]
-        }
-    ]
-}

main.go

@@ -1,7 +1,9 @@
 package main
 
 import (
+	"math/rand"
 	"os"
+	"time"
 
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/cmd"
@@ -17,5 +19,7 @@ func main() {
 
 	aptly.Version = Version
 
+	rand.Seed(time.Now().UnixNano())
+
 	os.Exit(cmd.Run(cmd.RootCommand(), os.Args[1:], true))
 }

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "March 2017" "" ""
+.TH "APTLY" "1" "July 2017" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -43,12 +43,30 @@ Configuration file is stored in JSON format (default values shown below):
   "dependencyFollowRecommends": false,
   "dependencyFollowAllVariants": false,
   "dependencyFollowSource": false,
+  "dependencyVerboseResolve": false,
   "gpgDisableSign": false,
   "gpgDisableVerify": false,
+  "gpgProvider": "gpg",
   "downloadSourcePackages": false,
+  "skipLegacyPool": true,
   "ppaDistributorID": "ubuntu",
   "ppaCodename": "",
   "skipContentsPublishing": false,
+  "FileSystemPublishEndpoints": {
+    "test1": {
+      "rootDir": "/opt/srv1/aptly_public",
+      "linkMethod": "symlink"
+    },
+    "test2": {
+      "rootDir": "/opt/srv2/aptly_public",
+      "linkMethod": "copy",
+      "verifyMethod": "md5"
+    },
+    "test3": {
+      "rootDir": "/opt/srv3/aptly_public",
+      "linkMethod": "hardlink"
+    }
+  },
   "S3PublishEndpoints": {
     "test": {
       "region": "us\-east\-1",
@@ -88,7 +106,7 @@ Options:
 .
 .TP
 \fBrootDir\fR
-is root of directory storage to store database (\fBrootDir\fR/db), downloaded packages (\fBrootDir\fR/pool) and published repositories (\fBrootDir\fR/public)
+is root of directory storage to store database (\fBrootDir\fR/db), downloaded packages (\fBrootDir\fR/pool) and the default for published repositories (\fBrootDir\fR/public)
 .
 .TP
 \fBdownloadConcurrency\fR
@@ -119,6 +137,10 @@ when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants a
 follow dependency from binary package to source package
 .
 .TP
+\fBdependencyVerboseResolve\fR
+print additional details while resolving dependencies (useful for debugging)
+.
+.TP
 \fBgpgDisableSign\fR
 don\(cqt sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing
 .
@@ -127,14 +149,26 @@ don\(cqt sign published repositories with gpg(1), also can be disabled on per\-r
 don\(cqt verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors
 .
 .TP
+\fBgpgProvider\fR
+implementation of PGP signing/validation \- \fBgpg\fR for external \fBgpg\fR utility or \fBinternal\fR to use Go internal implementation
+.
+.TP
 \fBdownloadSourcePackages\fR
 if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag
 .
 .TP
+\fBskipLegacyPool\fR
+in aptly up to version 1\.0\.0, package files were stored in internal package pool with MD5\-dervied path, since 1\.1\.0 package pool layout was changed; if option is enabled, aptly stops checking for legacy paths; by default option is enabled for new aptly installations and disabled when upgrading from older versions
+.
+.TP
 \fBppaDistributorID\fR, \fBppaCodename\fR
 specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
 .
 .TP
+\fBFileSystemPublishEndpoints\fR
+configuration of local filesystem publishing endpoints (see below)
+.
+.TP
 \fBS3PublishEndpoints\fR
 configuration of Amazon S3 publishing endpoints (see below)
 .
@@ -142,6 +176,27 @@ configuration of Amazon S3 publishing endpoints (see below)
 \fBSwiftPublishEndpoints\fR
 configuration of OpenStack Swift publishing endpoints (see below)
 .
+.SH "FILESYSTEM PUBLISHING ENDPOINTS"
+aptly defaults to publish to a single publish directory under \fBrootDir\fR/public\. For a more advanced publishing strategy, you can define one or more filesystem endpoints in the \fBFileSystemPublishEndpoints\fR list of the aptly configuration file\. Each endpoint has a name and the following associated settings:
+.
+.TP
+\fBrootDir\fR
+The publish directory, e\.g\., \fB/opt/srv/aptly_public\fR\.
+.
+.TP
+\fBlinkMethod\fR
+This is one of \fBhardlink\fR, \fBsymlink\fR or \fBcopy\fR\. It specifies how aptly links the files from the internal pool to the published directory\. If not specified, empty or wrong, this defaults to \fBhardlink\fR\.
+.
+.TP
+\fBverifyMethod\fR
+This is used only when setting the \fBlinkMethod\fR to \fBcopy\fR\. Possible values are \fBmd5\fR and \fBsize\fR\. It specifies how aptly compares existing links from the internal pool to the published directory\. The \fBsize\fR method compares only the file sizes, whereas the \fBmd5\fR method calculates the md5 checksum of the found file and compares it to the desired one\. If not specified, empty or wrong, this defaults to \fBmd5\fR\.
+.
+.P
+In order to publish to such an endpoint, specify the endpoint as \fBfilesystem:endpoint\-name\fR with \fBendpoint\-name\fR as the name given in the aptly configuration file\. For example:
+.
+.P
+\fBaptly publish snapshot wheezy\-main filesystem:test1:wheezy/daily\fR
+.
 .SH "S3 PUBLISHING ENDPOINTS"
 aptly could be configured to publish repository directly to Amazon S3 (or S3\-compatible cloud storage)\. First, publishing endpoints should be described in aptly configuration file\. Each endpoint has name and associated settings:
 .
@@ -185,6 +240,14 @@ bucket name
 \fBdisableMultiDel\fR
 (optional) for S3\-compatible cloud storages which do not support \fBMultiDel\fR S3 API, enable this setting (file deletion would be slower with this setting enabled)
 .
+.TP
+\fBforceSigV2\fR
+(optional) disable Signature V4 support, useful with non\-AWS S3\-compatible object stores which do not support SigV4, shouldn\(cqt be enabled for AWS
+.
+.TP
+\fBdebug\fR
+(optional) enables detailed request/response dump for each S3 operation
+.
 .P
 In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
 .
@@ -346,6 +409,10 @@ list of architectures to consider during (comma\-separated), default to all avai
 location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptly\.conf)
 .
 .TP
+\-\fBdb\-open\-attempts\fR=10
+number of attempts to open DB if it\(cqs locked by other instance
+.
+.TP
 \-\fBdep\-follow\-all\-variants\fR=false
 when processing dependencies, follow a & b if dependency is \(cqa|b\(cq
 .
@@ -361,6 +428,14 @@ when processing dependencies, follow from binary to Source packages
 \-\fBdep\-follow\-suggests\fR=false
 when processing dependencies, follow Suggests
 .
+.TP
+\-\fBdep\-verbose\-resolve\fR=false
+when processing dependencies, print detailed logs
+.
+.TP
+\-\fBgpg\-provider\fR=
+PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation)
+.
 .SH "CREATE NEW MIRROR"
 \fBaptly\fR \fBmirror\fR \fBcreate\fR \fIname\fR \fIarchive url\fR \fIdistribution\fR [\fIcomponent1\fR \|\.\|\.\|\.]
 .
@@ -510,6 +585,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 \-\fBmax\-tries\fR=1
 max download tries till process fails with download error
 .
+.TP
+\-\fBskip\-existing\-packages\fR=false
+do not check file existence for packages listed in the internal database of the mirror
+.
 .SH "RENAMES MIRROR"
 \fBaptly\fR \fBmirror\fR \fBrename\fR \fIold\-name\fR \fInew\-name\fR
 .
@@ -1282,6 +1361,10 @@ Options:
 run GPG with detached tty
 .
 .TP
+\-\fBbutautomaticupgrades\fR=
+set value for ButAutomaticUpgrades field
+.
+.TP
 \-\fBcomponent\fR=
 component name to publish (for multi\-component publishing, separate components with commas)
 .
@@ -1306,6 +1389,10 @@ GPG keyring to use (instead of default)
 label to publish
 .
 .TP
+\-\fBnotautomatic\fR=
+set value for NotAutomatic field
+.
+.TP
 \-\fBorigin\fR=
 origin name to publish
 .
@@ -1369,6 +1456,10 @@ Options:
 run GPG with detached tty
 .
 .TP
+\-\fBbutautomaticupgrades\fR=
+overwrite value for ButAutomaticUpgrades field
+.
+.TP
 \-\fBcomponent\fR=
 component name to publish (for multi\-component publishing, separate components with commas)
 .
@@ -1393,8 +1484,12 @@ GPG keyring to use (instead of default)
 label to publish
 .
 .TP
+\-\fBnotautomatic\fR=
+overwrite value for NotAutomatic field
+.
+.TP
 \-\fBorigin\fR=
-origin name to publish
+overwrite origin name to publish
 .
 .TP
 \-\fBpassphrase\fR=
@@ -1891,5 +1986,10 @@ Johannes Layher (https://github\.com/jola5)
 .IP "\[ci]" 4
 Charles Hsu (https://github\.com/charz)
 .
+.IP "\[ci]" 4
+Clemens Rabe (https://github\.com/seeraven)
+.
+.IP "\[ci]" 4
+TJ Merritt (https://github\.com/tjmerritt)
+.
 .IP "" 0
-

man/aptly.1.ronn.tmpl

@@ -35,12 +35,30 @@ Configuration file is stored in JSON format (default values shown below):
       "dependencyFollowRecommends": false,
       "dependencyFollowAllVariants": false,
       "dependencyFollowSource": false,
+      "dependencyVerboseResolve": false,
       "gpgDisableSign": false,
       "gpgDisableVerify": false,
+      "gpgProvider": "gpg",
       "downloadSourcePackages": false,
+      "skipLegacyPool": true,
       "ppaDistributorID": "ubuntu",
       "ppaCodename": "",
       "skipContentsPublishing": false,
+      "FileSystemPublishEndpoints": {
+        "test1": {
+          "rootDir": "/opt/srv1/aptly_public",
+          "linkMethod": "symlink"
+        },
+        "test2": {
+          "rootDir": "/opt/srv2/aptly_public",
+          "linkMethod": "copy",
+          "verifyMethod": "md5"
+        },
+        "test3": {
+          "rootDir": "/opt/srv3/aptly_public",
+          "linkMethod": "hardlink"
+        }
+      },
       "S3PublishEndpoints": {
         "test": {
           "region": "us-east-1",
@@ -75,7 +93,7 @@ Options:
 
   * `rootDir`:
     is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool) and
-    published repositories (`rootDir`/public)
+    the default for published repositories (`rootDir`/public)
 
   * `downloadConcurrency`:
     is a number of parallel download threads to use when downloading packages
@@ -99,6 +117,9 @@ Options:
   * `dependencyFollowSource`:
     follow dependency from binary package to source package
 
+  * `dependencyVerboseResolve`:
+    print additional details while resolving dependencies (useful for debugging)
+
   * `gpgDisableSign`:
     don't sign published repositories with gpg(1), also can be disabled on
     per-repo basis using `-skip-signing` flag when publishing
@@ -107,20 +128,60 @@ Options:
     don't verify remote mirrors with gpg(1), also can be disabled on
     per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors
 
+  * `gpgProvider`:
+    implementation of PGP signing/validation - `gpg` for external `gpg` utility or
+    `internal` to use Go internal implementation
+
   * `downloadSourcePackages`:
     if enabled, all mirrors created would have flag set to download source packages;
     this setting could be controlled on per-mirror basis with `-with-sources` flag
 
+  * `skipLegacyPool`:
+    in aptly up to version 1.0.0, package files were stored in internal package pool
+    with MD5-dervied path, since 1.1.0 package pool layout was changed;
+    if option is enabled, aptly stops checking for legacy paths;
+    by default option is enabled for new aptly installations and disabled when
+    upgrading from older versions
+
   * `ppaDistributorID`, `ppaCodename`:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
+  * `FileSystemPublishEndpoints`:
+    configuration of local filesystem publishing endpoints (see below)
+
   * `S3PublishEndpoints`:
     configuration of Amazon S3 publishing endpoints (see below)
 
   * `SwiftPublishEndpoints`:
     configuration of OpenStack Swift publishing endpoints (see below)
 
+## FILESYSTEM PUBLISHING ENDPOINTS
+
+aptly defaults to publish to a single publish directory under `rootDir`/public. For
+a more advanced publishing strategy, you can define one or more filesystem endpoints in the
+`FileSystemPublishEndpoints` list of the aptly configuration file. Each endpoint has a name
+and the following associated settings:
+
+   * `rootDir`:
+     The publish directory, e.g., `/opt/srv/aptly_public`.
+   * `linkMethod`:
+     This is one of `hardlink`, `symlink` or `copy`. It specifies how aptly links the
+     files from the internal pool to the published directory.
+     If not specified, empty or wrong, this defaults to `hardlink`.
+   * `verifyMethod`:
+     This is used only when setting the `linkMethod` to `copy`. Possible values are
+     `md5` and `size`. It specifies how aptly compares existing links from the
+     internal pool to the published directory. The `size` method compares only the
+     file sizes, whereas the `md5` method calculates the md5 checksum of the found
+     file and compares it to the desired one.
+     If not specified, empty or wrong, this defaults to `md5`.
+
+In order to publish to such an endpoint, specify the endpoint as `filesystem:endpoint-name`
+with `endpoint-name` as the name given in the aptly configuration file. For example:
+
+  `aptly publish snapshot wheezy-main filesystem:test1:wheezy/daily`
+
 ## S3 PUBLISHING ENDPOINTS
 
 aptly could be configured to publish repository directly to Amazon S3 (or S3-compatible
@@ -166,6 +227,11 @@ and associated settings:
    * `disableMultiDel`:
      (optional) for S3-compatible cloud storages which do not support `MultiDel` S3 API,
      enable this setting (file deletion would be slower with this setting enabled)
+   * `forceSigV2`:
+     (optional) disable Signature V4 support, useful with non-AWS S3-compatible object stores
+     which do not support SigV4, shouldn't be enabled for AWS
+   * `debug`:
+     (optional) enables detailed request/response dump for each S3 operation
 
 In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
 publishing prefix on the command line, e.g.:

manifest.json

@@ -1,31 +0,0 @@
-{
-    "dependencies": {
-        "github.com/gin-gonic/gin": {
-            "revision": "b1758d3bfa09e61ddbc1c9a627e936eec6a170de"
-        },
-        "github.com/h2non/filetype": {
-            "branch": "master"
-        },
-        "github.com/mkrautz/goar": {
-            "branch": "master"
-        },
-        "github.com/smira/go-uuid": {
-            "branch": "master"
-        },
-        "github.com/smira/go-xz": {
-            "branch": "master"
-        },
-        "github.com/ugorji/go": {
-            "revision": "71c2886f5a673a35f909803f38ece5810165097b"
-        },
-        "golang.org/x/crypto": {
-            "branch": "master"
-        },
-        "golang.org/x/sys": {
-            "branch": "master"
-        },
-        "gopkg.in/check.v1": {
-            "branch": "v1"
-        }
-    }
-}

pgp/1.signature

@@ -0,0 +1,17 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+iQIcBAABAgAGBQJY3WKsAAoJEK4Dls/4JTVAqs4P/2O21/A2/TB0d/wWfG4tPlVE
+p6+MpnPevspHtM3pGtD9z92WJO9K8BhjoUHwDh/GByqdt3mDXNccMlV/T5ZKP2It
+ZqHRLs5g1Sx6JBeARbyU1VXVx0Gkkji8EjIzR/0edZ7BXtPiFxf+dJWeOCHAz1VM
+NdBp/629r+FzGRn7vPW0Y8gOgNE6s2MeFlbw2apLT9CULdgt0Y6r5cv4eH6qaid5
+HEIZP5U98sBWsRP7SfRQKrjtHNGuZMsDDWuw83mhFOX4Lr5DvfZLKpuJYrOeGJTg
+O8Uu+gVQK0tSuN+YgdA0Q/Uahvlj0mgI3mkb9kmo19lISHF8bGVD/9zGeltqTzbv
+66nnpkXv132wqLElvZnsC6cwovpe9/2EpVGppFpw01D7Mrg+l8lO+FCMyLYaxNde
+6KtRDO7WSKSX2iNobrbbTlsw62OdgEyN3yfgaq7hN268MK4rd+mKEywUB5TXmbyo
+5HdnSGmL7LFbc2T5UzoYmSd6Y1wKCd1S8NeFSB0V2hMVm7UP8+3PDsyYwyzi1G+E
+n25+WOcl8qFX94aKXAxLwikzi1dPt3qDgfckiC2cJvc/nBN+A8KKfyf1M63Q4G/G
+mCZ1ehmWUqnpwImw6w0YJCjkiuY/zqCYiHdOmCtRe6iUmaejzKY+9oVF6A5XxB+P
+3+LnLWVzCmq2H/3PvxwJ
+=Rtpk
+-----END PGP SIGNATURE-----

pgp/1.text

@@ -0,0 +1,31 @@
+Architectures: all
+Codename: deb
+Components: contrib
+Date: Thu, 30 Mar 2017 19:55:24 UTC
+Label: PagerDuty, Inc.
+Origin: PagerDuty, Inc.
+Suite: stable
+MD5Sum:
+ 582e25878889dfecf25c00d459f4bdd7            13861 Packages
+ 49b1c5a52954278c4f06164bcfe95e50             2877 Packages.gz
+ db77040214a23ab564e3c3a09768dfea             2446 deb/Packages
+ 5799d88860e10251afb6737995811b3d              973 deb/Packages.gz
+ 20ba00172df962f951c70828c9fa8123              944 deb/Release
+SHA1:
+ 80e58103a3234f0f3b0920d1dd16d97f4ffc11a6            13861 Packages
+ 7e8916c13357363b0464cd5fc8fd469042ad6e23             2877 Packages.gz
+ 5a0cb9e0246a7c130a7be271fddd25cbba851a65             2446 deb/Packages
+ b661dbb9991d9ccf7c181426e86713dcfc70c609              973 deb/Packages.gz
+ f6f96166c00229f0c463b2c654ebd0a499bb084a              944 deb/Release
+SHA256:
+ a6c561644cdaa4dd1562698147f19e2cf3f6cd0efef172a68c1d164f1acb5095            13861 Packages
+ ba6abb2016c2e667affd07c80659acd1ddbdce3f36cd87e8dcff9824a62d65e1             2877 Packages.gz
+ 2b9133de7bd0ac7ac4dc9fa95a81c9cb9a3ccd9cdb6a945e4a3b05af110cf7f4             2446 deb/Packages
+ a4245f236c5fc3a6a53b25dff144433e9c648a08cca4ae86edd5e46aa14236da              973 deb/Packages.gz
+ 1ee2657183dd48cdbd30b87f373cf2ae9951c3970e7a4a5ef1e5a1a07b777427              944 deb/Release
+SHA512:
+ af258ba087feb8a9782075678d57d3706c9390a765fe8d34afee4143b5da83c16fdd8ea6f99f0b9979a9d77dc4db088c2aa7037576f0c5e29f1da3e9cd20bd05            13861 Packages
+ b3473ec7e4a1dced6daec0f6f477ec31c4eb8b7a174e068d259c21358728f2b9c5711f2c716055f524d478a057cbf78a710453fc0ba800d86fa1505a141fdf6b             2877 Packages.gz
+ 6e8f55f55489560094a5a98fc65ca2930c942e76f6e9ad924fe877c075ba08297d9e43163a0069224f6e8f5304943aac89c1ed45f34c9533b2b76d3c0a79ab27             2446 deb/Packages
+ e4e0a8440fd2198e358037c4b674ffcb245cbcf93a4f1bcd6158489b62d545a2078e9498e59180c27cbd2465c7941255275a8187f1bf119e80f8b227ce028d5f              973 deb/Packages.gz
+ b119366fef79f31f5e33876527dcec1d76037ebdcfc5c2380ba512e90b59db00bdd9fbceda977744be87a211df5f1c1fd5c41dafccef345d752a460f7fa54b3a              944 deb/Release

pgp/2.signature

@@ -0,0 +1,18 @@
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.11 (GNU/Linux)
+
+iEYEABEKAAYFAlNrkrEACgkQQJdur0N9BbV7RgCfbZGjC7ejdU5fMW6Kbk6bRQcS
+G2sAn1h7znlqgxolQOhYVAnsfmu96aTbiQIcBAABCgAGBQJTa5KxAAoJEDtP5qzA
+sh8yat4QALTR1k1DKijcCu9NHWm0p5iz6+cFOmUnYS8ewjhS3Oy5mk9WjXLTpOID
+BBykbsXnNIEpx4nvPhwX2jb/8XJNIT5pyhHDD7ydbQsDsQnhaah1gBwd5ZP3gwpF
+9IGJ15V4737rqeifYNKohn8//4GQsoIuhzyMOqIq8lIpOJyKzWvJm9ToW7kurF1d
+yQvB2rdXgOLUgXnpzsLu3Xw/p0bY+OUkdTxbfg+UxOIvwI1DYOPrTq/vPunMkA0C
+QuXv7yTdYiWWoV3IUqzF5iwY0nJAcfH6bBmyXXgr9WY9QXSw+CUjMfTI3EPCG8Rw
+8Z9z7LJ8zeH7DucaDkSVmPUE8uKPspc7CHuZ5b09O435TdbiargNAXwRNKKlEXcr
+1bQ2CZfve5jxKv3g7xEk4C/LpNMd/0w7DsqIuw6lRwoc4vNqdPlQMjywnHFNYTDl
+s5Tilg2T2pSE9SRRhLQtGAVP2VU5AD/WJfAUDHM5zLm9avZKsOphiTuXDJkaZxr7
+eMn1kQyzCh30ac9zJukh8PfEREY/BT8JFC7qWWUZ2zeevsOQZJ0WHL/lm6TZRsgX
+84qD7Z2UrTClnTNd6CUKHm6ispT9uC/BTFZ7efrw8mTPJotBNOpPNgmOVXFKsuoh
+SyHY769UhUN2MeCGjsLjee5jRg2moS421UmBZbeRgicH92BUaWzL
+=7r4e
+-----END PGP SIGNATURE-----