Aptly version 0.4.

coveralls.io is really unstable.

.gitignore

@@ -26,3 +26,9 @@ coverage.html
 coverage*.out
 
 *.pyc
+
+_vendor/
+
+gen
+man/aptly.1.html
+man/aptly.1.ronn

.travis.yml

@@ -3,6 +3,8 @@ language: go
 go:
   - 1.1
   - 1.2
+  - 1.2.1
+  - tip
 
 env:
   global:
@@ -12,4 +14,8 @@ install:
   - make prepare
 
 
-script: make travis
+script: make travis
+
+matrix:
+    allow_failures:
+        - go: tip

Gomfile

@@ -0,0 +1,22 @@
+gom 'code.google.com/p/go-uuid/uuid', :commit => '5fac954758f5'
+gom 'code.google.com/p/go.crypto/ssh/terminal', :commit => '7aa593ce8cea'
+gom 'code.google.com/p/gographviz', :commit => '212766062629'
+gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
+gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
+gom 'github.com/gonuts/commander', :commit => 'f8ba4e959ca914268227c3ebbd7f6bf0bb35541a'
+gom 'github.com/gonuts/flag', :commit => '741a6cbd37a30dedc93f817e7de6aaf0ca38a493'
+gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => '527a7b286bd095794af6c519627b7ed3d8fd067a'
+gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
+gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
+
+group :test do
+    gom 'launchpad.net/gocheck'
+end
+
+group :development do
+   gom 'github.com/golang/lint/golint'
+   gom 'github.com/mattn/goveralls'
+   gom 'github.com/axw/gocov/gocov'
+   gom 'code.google.com/p/go.tools/cmd/cover'
+end

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2013 Andrey Smirnov. All rights reserved.
+Copyright 2013-2014 Andrey Smirnov. All rights reserved.
 
 MIT License
 
@@ -14,8 +14,8 @@ all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 IN THE SOFTWARE.

Makefile

@@ -1,60 +1,64 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
+PACKAGES=database debian files http utils
+ALL_PACKAGES=aptly cmd console database debian files http utils
+BINPATH=$(abspath ./_vendor/bin)
+GOM_ENVIRONMENT=-test
+PYTHON?=python
 
-ifeq ($(TRAVIS), true)
-GOVERALLS?=$(HOME)/gopath/bin/goveralls
-SRCPATH?=$(HOME)/gopath/src
-BINPATH=$(HOME)/gopath/bin
-else
-GOVERALLS?=goveralls
-SRCPATH?=$(GOPATH)/src
-BINPATH?=$(GOPATH)/bin
-endif
-
-ifeq ($(GOVERSION), go1.2)
+ifeq ($(GOVERSION), devel)
 TRAVIS_TARGET=coveralls
-PREPARE_LIST=cover-prepare
+GOM_ENVIRONMENT+=-development
 else
 TRAVIS_TARGET=test
-PREPARE_LIST=
+endif
+
+ifeq ($(TRAVIS), true)
+GOM=$(HOME)/gopath/bin/gom
+else
+GOM=gom
 endif
 
 all: test check system-test
 
-prepare: $(PREPARE_LIST)
-	go get -d -v ./...
-	go get launchpad.net/gocheck
-
-cover-prepare:
-	go get github.com/golang/lint/golint
-	go get github.com/mattn/goveralls
-	go get github.com/axw/gocov/gocov
-	go get code.google.com/p/go.tools/cmd/cover
+prepare:
+	go get -u github.com/mattn/gom
+	$(GOM) $(GOM_ENVIRONMENT) install
 
 coverage.out:
-	go test -coverprofile=coverage.debian.out -covermode=count ./debian
-	go test -coverprofile=coverage.utils.out -covermode=count ./utils
-	go test -coverprofile=coverage.database.out -covermode=count ./database
+	rm -f coverage.*.out
+	for i in $(PACKAGES); do $(GOM) test -coverprofile=coverage.$$i.out -covermode=count ./$$i; done
 	echo "mode: count" > coverage.out
 	grep -v -h "mode: count" coverage.*.out >> coverage.out
+	rm -f coverage.*.out
 
 coverage: coverage.out
-	go tool cover -html=coverage.out
+	$(GOM) exec go tool cover -html=coverage.out
 	rm -f coverage.out
 
 check:
-	go tool vet -all=true .
-	golint .
+	$(GOM) exec go tool vet -all=true -shadow=true $(ALL_PACKAGES:%=./%)
+	$(GOM) exec golint $(ALL_PACKAGES:%=./%)
 
-system-test:
-	go install
-	PATH=$(BINPATH):$(PATH) python system/run.py
+install:
+	$(GOM) build -o $(BINPATH)/aptly
+
+system-test: install
+ifeq ($(GOVERSION),$(filter $(GOVERSION),go1.2 go1.2.1 devel))
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+endif
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	PATH=$(BINPATH)/:$(PATH) $(PYTHON) system/run.py --long
 
 travis: $(TRAVIS_TARGET) system-test
 
 test:
-	go test -v ./... -gocheck.v=true
+	$(GOM) test -v ./... -gocheck.v=true
 
 coveralls: coverage.out
-	@$(GOVERALLS) -service travis-ci.org -coverprofile=coverage.out $(COVERALLS_TOKEN)
+	$(GOM) exec $(BINPATH)/goveralls -service travis-ci.org -coverprofile=coverage.out -repotoken=$(COVERALLS_TOKEN)
+
+mem.png: mem.dat mem.gp
+	gnuplot mem.gp
+	open mem.png
 
 .PHONY: coverage.out

README.rst

@@ -19,24 +19,33 @@ Aptly features: ("+" means planned features)
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
-* merge two or more snapshots into one (+)
+* merge two or more snapshots into one
 * filter repository by search query, pulling dependencies when required (+)
 * publish self-made packages as Debian repositories (+)
+* mirror repositories "as-is" (without resigning with user's key) (+)
+* support for yum repositories (+)
 
 Current limitations:
 
-* source packages, debian-installer and translations not supported yet
-* checksums and signature are not verified while downloading
-* deleting created items is not implemented
-* cleaning up stale files is not implemented
-
-Currently aptly is under heavy development, so please use it with care.
+* debian-installer and translations not supported yet
 
 Download
 --------
 
 Binary executables (depends almost only on libc) are available for download from `Bintray <http://dl.bintray.com/smira/aptly/>`_.
 
-If you have Go environment set up, you can build aptly from source by running::
+If you have Go environment set up, you can build aptly from source by running (go 1.1+ required)::
+
+    go get -u github.com/mattn/gom
+    mkdir -p $GOPATH/src/github.com/smira/aptly
+    git clone https://github.com/smira/aptly $GOPATH/src/github.com/smira/aptly
+    cd $GOPATH/src/github.com/smira/aptly
+    gom install
+    gom build -o $GOPATH/bin/aptly
+
+Aptly is using `gom <https://github.com/mattn/gom>`_ to fix external dependencies, so regular ``go get github.com/smira/aptly``
+should work as well, but might fail or produce different result (if external libraries got updated).
+
+If you don't have Go installed (or older version), you can easily install Go using `gvm <https://github.com/moovweb/gvm/>`_.
+
 
-    go get github.com/smira/aptly

aptly/interfaces.go

@@ -0,0 +1,82 @@
+// Package aptly provides common infrastructure that doesn't depend directly on
+// Debian or CentOS
+package aptly
+
+import (
+	"github.com/smira/aptly/utils"
+	"io"
+	"os"
+)
+
+// PackagePool is asbtraction of package pool storage.
+//
+// PackagePool stores all the package files, deduplicating them.
+type PackagePool interface {
+	// Path returns full path to package file in pool given any name and hash of file contents
+	Path(filename string, hashMD5 string) (string, error)
+	// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+	RelativePath(filename string, hashMD5 string) (string, error)
+	// FilepathList returns file paths of all the files in the pool
+	FilepathList(progress Progress) ([]string, error)
+	// Remove deletes file in package pool returns its size
+	Remove(path string) (size int64, err error)
+	// Import copies file into package pool
+	Import(path string, hashMD5 string) error
+}
+
+// PublishedStorage is abstraction of filesystem storing all published repositories
+type PublishedStorage interface {
+	// PublicPath returns root of public part
+	PublicPath() string
+	// MkDir creates directory recursively under public path
+	MkDir(path string) error
+	// CreateFile creates file for writing under public path
+	CreateFile(path string) (*os.File, error)
+	// RemoveDirs removes directory structure under public path
+	RemoveDirs(path string) error
+	// LinkFromPool links package file from pool to dist's pool location
+	LinkFromPool(prefix string, component string, poolDirectory string, sourcePool PackagePool, sourcePath string) (string, error)
+	// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
+	ChecksumsForFile(path string) (utils.ChecksumInfo, error)
+}
+
+// Progress is a progress displaying entity, it allows progress bars & simple prints
+type Progress interface {
+	// Writer interface to support progress bar ticking
+	io.Writer
+	// Start makes progress start its work
+	Start()
+	// Shutdown shuts down progress display
+	Shutdown()
+	// Flush returns when all queued messages are sent
+	Flush()
+	// InitBar starts progressbar for count bytes or count items
+	InitBar(count int64, isBytes bool)
+	// ShutdownBar stops progress bar and hides it
+	ShutdownBar()
+	// AddBar increments progress for progress bar
+	AddBar(count int)
+	// SetBar sets current position for progress bar
+	SetBar(count int)
+	// Printf does printf but in safe manner: not overwriting progress bar
+	Printf(msg string, a ...interface{})
+	// ColoredPrintf does printf in colored way + newline
+	ColoredPrintf(msg string, a ...interface{})
+}
+
+// Downloader is parallel HTTP fetcher
+type Downloader interface {
+	// Download starts new download task
+	Download(url string, destination string, result chan<- error)
+	// DownloadWithChecksum starts new download task with checksum verification
+	DownloadWithChecksum(url string, destination string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool)
+	// Pause pauses task processing
+	Pause()
+	// Resume resumes task processing
+	Resume()
+	// Shutdown stops downloader after current tasks are finished,
+	// but doesn't process rest of queue
+	Shutdown()
+	// GetProgress returns Progress object
+	GetProgress() Progress
+}

aptly/version.go

@@ -0,0 +1,7 @@
+package aptly
+
+// Version of aptly
+const Version = "0.4"
+
+// Enable debugging features?
+const EnableDebug = false

cmd/cmd.go

@@ -0,0 +1,83 @@
+// Package cmd implements console commands
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/debian"
+	"os"
+	"time"
+)
+
+// ListPackagesRefList shows list of packages in PackageRefList
+func ListPackagesRefList(reflist *debian.PackageRefList) (err error) {
+	fmt.Printf("Packages:\n")
+
+	if reflist == nil {
+		return
+	}
+
+	packageCollection := debian.NewPackageCollection(context.database)
+
+	err = reflist.ForEach(func(key []byte) error {
+		p, err := packageCollection.ByKey(key)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("  %s\n", p)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	return
+}
+
+// RootCommand creates root command in command tree
+func RootCommand() *commander.Command {
+	cmd := &commander.Command{
+		UsageLine: os.Args[0],
+		Short:     "Debian repository management tool",
+		Long: `
+aptly is a tool to create partial and full mirrors of remote
+repositories, manage local repositories, filter them, merge,
+upgrade individual packages, take snapshots and publish them
+back as Debian repositories.
+
+aptly goal is to establish repeatiblity and controlled changes
+in package environment. aptly allows to fix set of packages in
+repository, so that package installation and upgrade becomes
+deterministic. At the same time aptly allows to perform controlled,
+fine-grained changes in repository contents to transition your
+package environment to new version.`,
+		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
+		Subcommands: []*commander.Command{
+			makeCmdDb(),
+			makeCmdGraph(),
+			makeCmdMirror(),
+			makeCmdRepo(),
+			makeCmdServe(),
+			makeCmdSnapshot(),
+			makeCmdPublish(),
+			makeCmdVersion(),
+		},
+	}
+
+	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
+	cmd.Flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
+	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
+	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
+	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
+	cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+
+	if aptly.EnableDebug {
+		cmd.Flag.String("cpuprofile", "", "write cpu profile to file")
+		cmd.Flag.String("memprofile", "", "write memory profile to this file")
+		cmd.Flag.String("memstats", "", "write memory stats periodically to this file")
+		cmd.Flag.Duration("meminterval", 100*time.Millisecond, "memory stats dump interval")
+	}
+	return cmd
+}

cmd/context.go

@@ -0,0 +1,151 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"strings"
+	"time"
+)
+
+// Common context shared by all commands
+var context struct {
+	progress          aptly.Progress
+	downloader        aptly.Downloader
+	database          database.Storage
+	packagePool       aptly.PackagePool
+	publishedStorage  aptly.PublishedStorage
+	dependencyOptions int
+	architecturesList []string
+	// Debug features
+	fileCPUProfile *os.File
+	fileMemProfile *os.File
+	fileMemStats   *os.File
+}
+
+// InitContext initializes context with default settings
+func InitContext(cmd *commander.Command) error {
+	var err error
+
+	context.dependencyOptions = 0
+	if utils.Config.DepFollowSuggests || cmd.Flag.Lookup("dep-follow-suggests").Value.Get().(bool) {
+		context.dependencyOptions |= debian.DepFollowSuggests
+	}
+	if utils.Config.DepFollowRecommends || cmd.Flag.Lookup("dep-follow-recommends").Value.Get().(bool) {
+		context.dependencyOptions |= debian.DepFollowRecommends
+	}
+	if utils.Config.DepFollowAllVariants || cmd.Flag.Lookup("dep-follow-all-variants").Value.Get().(bool) {
+		context.dependencyOptions |= debian.DepFollowAllVariants
+	}
+	if utils.Config.DepFollowSource || cmd.Flag.Lookup("dep-follow-source").Value.Get().(bool) {
+		context.dependencyOptions |= debian.DepFollowSource
+	}
+
+	context.architecturesList = utils.Config.Architectures
+	optionArchitectures := cmd.Flag.Lookup("architectures").Value.String()
+	if optionArchitectures != "" {
+		context.architecturesList = strings.Split(optionArchitectures, ",")
+	}
+
+	context.progress = console.NewProgress()
+	context.progress.Start()
+
+	context.downloader = http.NewDownloader(utils.Config.DownloadConcurrency, context.progress)
+
+	context.database, err = database.OpenDB(filepath.Join(utils.Config.RootDir, "db"))
+	if err != nil {
+		return fmt.Errorf("can't open database: %s", err)
+	}
+
+	context.packagePool = files.NewPackagePool(utils.Config.RootDir)
+	context.publishedStorage = files.NewPublishedStorage(utils.Config.RootDir)
+
+	if aptly.EnableDebug {
+		cpuprofile := cmd.Flag.Lookup("cpuprofile").Value.String()
+		if cpuprofile != "" {
+			context.fileCPUProfile, err = os.Create(cpuprofile)
+			if err != nil {
+				return err
+			}
+			pprof.StartCPUProfile(context.fileCPUProfile)
+		}
+
+		memprofile := cmd.Flag.Lookup("memprofile").Value.String()
+		if memprofile != "" {
+			context.fileMemProfile, err = os.Create(memprofile)
+			if err != nil {
+				return err
+			}
+		}
+
+		memstats := cmd.Flag.Lookup("memstats").Value.String()
+		if memstats != "" {
+			interval := cmd.Flag.Lookup("meminterval").Value.Get().(time.Duration)
+
+			context.fileMemStats, err = os.Create(memstats)
+			if err != nil {
+				return err
+			}
+
+			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+
+			go func() {
+				var stats runtime.MemStats
+
+				start := time.Now().UnixNano()
+
+				for {
+					runtime.ReadMemStats(&stats)
+					if context.fileMemStats != nil {
+						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
+						time.Sleep(interval)
+					} else {
+						break
+					}
+				}
+			}()
+		}
+	}
+
+	return nil
+}
+
+// ShutdownContext shuts context down
+func ShutdownContext() {
+	if aptly.EnableDebug {
+		if context.fileMemProfile != nil {
+			pprof.WriteHeapProfile(context.fileMemProfile)
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+		if context.fileCPUProfile != nil {
+			pprof.StopCPUProfile()
+			context.fileCPUProfile.Close()
+			context.fileCPUProfile = nil
+		}
+		if context.fileMemProfile != nil {
+			context.fileMemProfile.Close()
+			context.fileMemProfile = nil
+		}
+	}
+	if context.database != nil {
+		context.database.Close()
+	}
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+	}
+}

cmd/db.go

@@ -0,0 +1,17 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+)
+
+func makeCmdDb() *commander.Command {
+	return &commander.Command{
+		UsageLine: "db",
+		Short:     "manage aptly's internal database and package pool",
+		Subcommands: []*commander.Command{
+			makeCmdDbCleanup(),
+		},
+		Flag: *flag.NewFlagSet("aptly-db", flag.ExitOnError),
+	}
+}

cmd/db_cleanup.go

@@ -0,0 +1,166 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"sort"
+)
+
+// aptly db cleanup
+func aptlyDbCleanup(cmd *commander.Command, args []string) error {
+	var err error
+
+	if len(args) != 0 {
+		cmd.Usage()
+		return err
+	}
+
+	// collect information about references packages...
+	existingPackageRefs := debian.NewPackageRefList()
+
+	context.progress.Printf("Loading mirrors, local repos and snapshots...\n")
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+		err := repoCollection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
+		err := localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+		err := snapshotCollection.LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false)
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	// ... and compare it to the list of all packages
+	context.progress.Printf("Loading list of all packages...\n")
+	packageCollection := debian.NewPackageCollection(context.database)
+	allPackageRefs := packageCollection.AllPackageRefs()
+
+	toDelete := allPackageRefs.Substract(existingPackageRefs)
+
+	// delete packages that are no longer referenced
+	context.progress.Printf("Deleting unreferenced packages (%d)...\n", toDelete.Len())
+
+	context.database.StartBatch()
+	err = toDelete.ForEach(func(ref []byte) error {
+		return packageCollection.DeleteByKey(ref)
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.database.FinishBatch()
+	if err != nil {
+		return fmt.Errorf("unable to write to DB: %s", err)
+	}
+
+	// now, build a list of files that should be present in Repository (package pool)
+	context.progress.Printf("Building list of files referenced by packages...\n")
+	referencedFiles := make([]string, 0, existingPackageRefs.Len())
+	context.progress.InitBar(int64(existingPackageRefs.Len()), false)
+
+	err = existingPackageRefs.ForEach(func(key []byte) error {
+		pkg, err := packageCollection.ByKey(key)
+		if err != nil {
+			return err
+		}
+		paths, err := pkg.FilepathList(context.packagePool)
+		if err != nil {
+			return err
+		}
+		referencedFiles = append(referencedFiles, paths...)
+		context.progress.AddBar(1)
+
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(referencedFiles)
+	context.progress.ShutdownBar()
+
+	// build a list of files in the package pool
+	context.progress.Printf("Building list of files in package pool...\n")
+	existingFiles, err := context.packagePool.FilepathList(context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to collect file paths: %s", err)
+	}
+
+	// find files which are in the pool but not referenced by packages
+	filesToDelete := utils.StrSlicesSubstract(existingFiles, referencedFiles)
+
+	// delete files that are no longer referenced
+	context.progress.Printf("Deleting unreferenced files (%d)...\n", len(filesToDelete))
+
+	if len(filesToDelete) > 0 {
+		context.progress.InitBar(int64(len(filesToDelete)), false)
+		totalSize := int64(0)
+		for _, file := range filesToDelete {
+			size, err := context.packagePool.Remove(file)
+			if err != nil {
+				return err
+			}
+
+			context.progress.AddBar(1)
+			totalSize += size
+		}
+		context.progress.ShutdownBar()
+
+		context.progress.Printf("Disk space freed: %.2f GiB...\n", float64(totalSize)/1024.0/1024.0/1024.0)
+	}
+	return err
+}
+
+func makeCmdDbCleanup() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyDbCleanup,
+		UsageLine: "cleanup",
+		Short:     "cleanup DB and package pool",
+		Long: `
+Database cleanup removes information about unreferenced packages and removes
+files in the package pool that aren't used by packages anymore
+
+Example:
+
+  $ aptly db cleanup
+`,
+		Flag: *flag.NewFlagSet("aptly-db-cleanup", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/graph.go

@@ -0,0 +1,211 @@
+package cmd
+
+import (
+	"bytes"
+	"code.google.com/p/gographviz"
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+)
+
+func graphvizEscape(s string) string {
+	return fmt.Sprintf("\"%s\"", strings.Replace(s, "\"", "\\\"", 0))
+}
+
+func aptlyGraph(cmd *commander.Command, args []string) error {
+	var err error
+
+	graph := gographviz.NewGraph()
+	graph.SetDir(true)
+	graph.SetName("aptly")
+
+	existingNodes := map[string]bool{}
+
+	fmt.Printf("Loading mirrors...\n")
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+
+	err = repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+		err := repoCollection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkgoldenrod1",
+			"label": graphvizEscape(fmt.Sprintf("{Mirror %s|url: %s|dist: %s|comp: %s|arch: %s|pkgs: %d}",
+				repo.Name, repo.ArchiveRoot, repo.Distribution, strings.Join(repo.Components, ", "),
+				strings.Join(repo.Architectures, ", "), repo.NumPackages())),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loading local repos...\n")
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+
+	err = localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
+		err := localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "mediumseagreen",
+			"label": graphvizEscape(fmt.Sprintf("{Repo %s|comment: %s|pkgs: %d}",
+				repo.Name, repo.Comment, repo.NumPackages())),
+		})
+		existingNodes[repo.UUID] = true
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loading snapshots...\n")
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+		existingNodes[snapshot.UUID] = true
+		return nil
+	})
+
+	err = snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+		err := snapshotCollection.LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		description := snapshot.Description
+		if snapshot.SourceKind == "repo" {
+			description = "Snapshot from repo"
+		}
+
+		graph.AddNode("aptly", graphvizEscape(snapshot.UUID), map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "cadetblue1",
+			"label":     graphvizEscape(fmt.Sprintf("{Snapshot %s|%s|pkgs: %d}", snapshot.Name, description, snapshot.NumPackages())),
+		})
+
+		if snapshot.SourceKind == "repo" || snapshot.SourceKind == "local" || snapshot.SourceKind == "snapshot" {
+			for _, uuid := range snapshot.SourceIDs {
+				_, exists := existingNodes[uuid]
+				if exists {
+					graph.AddEdge(graphvizEscape(uuid), "", graphvizEscape(snapshot.UUID), "", true, nil)
+				}
+			}
+		}
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("Loading published repos...\n")
+
+	publishedCollection := debian.NewPublishedRepoCollection(context.database)
+
+	publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
+		graph.AddNode("aptly", graphvizEscape(repo.UUID), map[string]string{
+			"shape":     "Mrecord",
+			"style":     "filled",
+			"fillcolor": "darkolivegreen1",
+			"label":     graphvizEscape(fmt.Sprintf("{Published %s/%s|comp: %s|arch: %s}", repo.Prefix, repo.Distribution, repo.Component, strings.Join(repo.Architectures, ", "))),
+		})
+
+		_, exists := existingNodes[repo.SnapshotUUID]
+		if exists {
+			graph.AddEdge(graphvizEscape(repo.SnapshotUUID), "", graphvizEscape(repo.UUID), "", true, nil)
+		}
+
+		return nil
+	})
+
+	fmt.Printf("Generating graph...\n")
+
+	buf := bytes.NewBufferString(graph.String())
+
+	tempfile, err := ioutil.TempFile("", "aptly-graph")
+	if err != nil {
+		return err
+	}
+	tempfile.Close()
+	os.Remove(tempfile.Name())
+
+	tempfilename := tempfile.Name() + ".png"
+
+	command := exec.Command("dot", "-Tpng", "-o"+tempfilename)
+	command.Stderr = os.Stderr
+
+	stdin, err := command.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	err = command.Start()
+	if err != nil {
+		return fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err)
+	}
+
+	_, err = io.Copy(stdin, buf)
+	if err != nil {
+		return err
+	}
+
+	err = stdin.Close()
+	if err != nil {
+		return err
+	}
+
+	err = command.Wait()
+	if err != nil {
+		return err
+	}
+
+	err = exec.Command("open", tempfilename).Run()
+	if err != nil {
+		fmt.Printf("Rendered to PNG file: %s\n", tempfilename)
+		err = nil
+	}
+
+	return err
+}
+
+func makeCmdGraph() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyGraph,
+		UsageLine: "graph",
+		Short:     "render graph of relationships",
+		Long: `
+Command graph displays relationship between mirrors, local repositories,
+snapshots and published repositories using graphviz package to render
+graph as image.
+
+Example:
+
+  $ aptly graph
+`,
+		Flag: *flag.NewFlagSet("aptly-graph", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/mirror.go

@@ -0,0 +1,60 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+func getVerifier(cmd *commander.Command) (utils.Verifier, error) {
+	if utils.Config.GpgDisableVerify || cmd.Flag.Lookup("ignore-signatures").Value.Get().(bool) {
+		return nil, nil
+	}
+
+	verifier := &utils.GpgVerifier{}
+	for _, keyRing := range keyRings.keyRings {
+		verifier.AddKeyring(keyRing)
+	}
+
+	err := verifier.InitKeyring()
+	if err != nil {
+		return nil, err
+	}
+
+	return verifier, nil
+}
+
+type keyRingsFlag struct {
+	keyRings []string
+}
+
+func (k *keyRingsFlag) Set(value string) error {
+	k.keyRings = append(k.keyRings, value)
+	return nil
+}
+
+func (k *keyRingsFlag) Get() interface{} {
+	return k.keyRings
+}
+
+func (k *keyRingsFlag) String() string {
+	return strings.Join(k.keyRings, ",")
+}
+
+var keyRings = keyRingsFlag{}
+
+func makeCmdMirror() *commander.Command {
+	return &commander.Command{
+		UsageLine: "mirror",
+		Short:     "manage mirrors of remote repositories",
+		Subcommands: []*commander.Command{
+			makeCmdMirrorCreate(),
+			makeCmdMirrorList(),
+			makeCmdMirrorShow(),
+			makeCmdMirrorDrop(),
+			makeCmdMirrorUpdate(),
+		},
+		Flag: *flag.NewFlagSet("aptly-mirror", flag.ExitOnError),
+	}
+}

cmd/mirror_create.go

@@ -0,0 +1,87 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
+	var err error
+	if !(len(args) == 2 && strings.HasPrefix(args[1], "ppa:") || len(args) >= 3) {
+		cmd.Usage()
+		return err
+	}
+
+	downloadSources := utils.Config.DownloadSourcePackages || cmd.Flag.Lookup("with-sources").Value.Get().(bool)
+
+	var (
+		mirrorName, archiveURL, distribution string
+		components                           []string
+	)
+
+	mirrorName = args[0]
+	if len(args) == 2 {
+		archiveURL, distribution, components, err = debian.ParsePPA(args[1])
+		if err != nil {
+			return err
+		}
+	} else {
+		archiveURL, distribution, components = args[1], args[2], args[3:]
+	}
+
+	repo, err := debian.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.architecturesList, downloadSources)
+	if err != nil {
+		return fmt.Errorf("unable to create mirror: %s", err)
+	}
+
+	verifier, err := getVerifier(cmd)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	err = repo.Fetch(context.downloader, verifier)
+	if err != nil {
+		return fmt.Errorf("unable to fetch mirror: %s", err)
+	}
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+
+	err = repoCollection.Add(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add mirror: %s", err)
+	}
+
+	fmt.Printf("\nMirror %s successfully added.\nYou can run 'aptly mirror update %s' to download repository contents.\n", repo, repo.Name)
+	return err
+}
+
+func makeCmdMirrorCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorCreate,
+		UsageLine: "create <name> <archive url> <distribution> [<component1> ...]",
+		Short:     "create new mirror",
+		Long: `
+Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
+via HTTP. aptly would try download Release file from remote repository and verify its signature.
+
+PPA urls could specified in short format:
+
+  $ aptly mirror create <name> ppa:<user>/<project>
+
+Example:
+
+  $ aptly mirror create wheezy-main http://mirror.yandex.ru/debian/ wheezy main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-create", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/mirror_drop.go

@@ -0,0 +1,70 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := repoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshotCollection := debian.NewSnapshotCollection(context.database)
+		snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+
+		if len(snapshots) > 0 {
+			fmt.Printf("Mirror `%s` was used to create following snapshots:\n", repo.Name)
+			for _, snapshot := range snapshots {
+				fmt.Printf(" * %s\n", snapshot)
+			}
+
+			return fmt.Errorf("won't delete mirror with snapshots, use -force to override")
+		}
+	}
+
+	err = repoCollection.Drop(repo)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Mirror `%s` has been removed.\n", repo.Name)
+
+	return err
+}
+
+func makeCmdMirrorDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete mirror",
+		Long: `
+Drop deletes information about remote repository mirror <name>. Package data is not deleted
+(it could be still used by other mirrors or snapshots).  If mirror is used as source
+to create a snapshot, aptly would refuse to delete such mirror, use flag -force to override.
+
+Example:
+
+  $ aptly mirror drop wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "force mirror deletion even if used by snapshots")
+
+	return cmd
+}

cmd/mirror_list.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlyMirrorList(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 0 {
+		cmd.Usage()
+		return err
+	}
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+
+	if repoCollection.Len() > 0 {
+		fmt.Printf("List of mirrors:\n")
+		repos := make([]string, repoCollection.Len())
+		i := 0
+		repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
+			repos[i] = repo.String()
+			i++
+			return nil
+		})
+
+		sort.Strings(repos)
+		for _, repo := range repos {
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
+	} else {
+		fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
+	}
+	return err
+}
+
+func makeCmdMirrorList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorList,
+		UsageLine: "list",
+		Short:     "list mirrors",
+		Long: `
+List shows full list of remote repository mirrors.
+
+Example:
+
+  $ aptly mirror list
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-list", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/mirror_show.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+func aptlyMirrorShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := repoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = repoCollection.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", repo.Name)
+	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
+	fmt.Printf("Distribution: %s\n", repo.Distribution)
+	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
+	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
+	downloadSources := "no"
+	if repo.DownloadSources {
+		downloadSources = "yes"
+	}
+	fmt.Printf("Download Sources: %s\n", downloadSources)
+	if repo.LastDownloadDate.IsZero() {
+		fmt.Printf("Last update: never\n")
+	} else {
+		fmt.Printf("Last update: %s\n", repo.LastDownloadDate.Format("2006-01-02 15:04:05 MST"))
+		fmt.Printf("Number of packages: %d\n", repo.NumPackages())
+	}
+
+	fmt.Printf("\nInformation from release file:\n")
+	for _, k := range utils.StrMapSortedKeys(repo.Meta) {
+		fmt.Printf("%s: %s\n", k, repo.Meta[k])
+	}
+
+	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		if repo.LastDownloadDate.IsZero() {
+			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
+		} else {
+			ListPackagesRefList(repo.RefList())
+		}
+	}
+
+	return err
+}
+
+func makeCmdMirrorShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorShow,
+		UsageLine: "show <name>",
+		Short:     "show details about mirror",
+		Long: `
+Shows detailed information about mirror.
+
+Example:
+
+  $ aptly mirror show wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-packages", false, "show detailed list of packages and versions stored in the mirror")
+
+	return cmd
+}

cmd/mirror_update.go

@@ -0,0 +1,80 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	repoCollection := debian.NewRemoteRepoCollection(context.database)
+	repo, err := repoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = repoCollection.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	ignoreMismatch := cmd.Flag.Lookup("ignore-checksums").Value.Get().(bool)
+
+	verifier, err := getVerifier(cmd)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG verifier: %s", err)
+	}
+
+	err = repo.Fetch(context.downloader, verifier)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	packageCollection := debian.NewPackageCollection(context.database)
+
+	err = repo.Download(context.progress, context.downloader, packageCollection, context.packagePool, ignoreMismatch)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = repoCollection.Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	context.progress.Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
+	return err
+}
+
+func makeCmdMirrorUpdate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyMirrorUpdate,
+		UsageLine: "update <name>",
+		Short:     "update mirror",
+		Long: `
+Updates remote mirror (downloads package files and meta information). When mirror is created,
+this command should be run for the first time to fetch mirror contents. This command could be
+run many times to get updated repository contents. If interrupted, command could be restarted safely.
+
+Example:
+
+  $ aptly mirror update wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
+	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Var(&keyRings, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")
+
+	return cmd
+}

cmd/publish.go

@@ -0,0 +1,38 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/utils"
+)
+
+func getSigner(cmd *commander.Command) (utils.Signer, error) {
+	if cmd.Flag.Lookup("skip-signing").Value.Get().(bool) || utils.Config.GpgDisableSign {
+		return nil, nil
+	}
+
+	signer := &utils.GpgSigner{}
+	signer.SetKey(cmd.Flag.Lookup("gpg-key").Value.String())
+	signer.SetKeyRing(cmd.Flag.Lookup("keyring").Value.String(), cmd.Flag.Lookup("secret-keyring").Value.String())
+
+	err := signer.Init()
+	if err != nil {
+		return nil, err
+	}
+
+	return signer, nil
+
+}
+
+func makeCmdPublish() *commander.Command {
+	return &commander.Command{
+		UsageLine: "publish",
+		Short:     "manage published repositories",
+		Subcommands: []*commander.Command{
+			makeCmdPublishSnapshot(),
+			makeCmdPublishList(),
+			makeCmdPublishDrop(),
+		},
+		Flag: *flag.NewFlagSet("aptly-publish", flag.ExitOnError),
+	}
+}

cmd/publish_drop.go

@@ -0,0 +1,53 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyPublishDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return err
+	}
+
+	distribution := args[0]
+	prefix := "."
+
+	if len(args) == 2 {
+		prefix = args[1]
+	}
+
+	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+
+	err = publishedCollecton.Remove(context.publishedStorage, prefix, distribution)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	fmt.Printf("\nPublished repositroy has been removed successfully.\n")
+
+	return err
+}
+
+func makeCmdPublishDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishDrop,
+		UsageLine: "drop <distribution> [<prefix>]",
+		Short:     "remove published repository",
+		Long: `
+Command removes whatever has been published under specified <prefix> and
+<distribution> name.
+
+Example:
+
+    $ aptly publish drop wheezy
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-drop", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/publish_list.go

@@ -0,0 +1,69 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlyPublishList(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 0 {
+		cmd.Usage()
+		return err
+	}
+
+	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	if publishedCollecton.Len() == 0 {
+		fmt.Printf("No snapshots have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
+		return err
+	}
+
+	published := make([]string, 0, publishedCollecton.Len())
+
+	err = publishedCollecton.ForEach(func(repo *debian.PublishedRepo) error {
+		err := publishedCollecton.LoadComplete(repo, snapshotCollection)
+		if err != nil {
+			return err
+		}
+
+		published = append(published, repo.String())
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to load list of repos: %s", err)
+	}
+
+	sort.Strings(published)
+
+	fmt.Printf("Published repositories:\n")
+
+	for _, description := range published {
+		fmt.Printf("  * %s\n", description)
+	}
+
+	return err
+}
+
+func makeCmdPublishList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishList,
+		UsageLine: "list",
+		Short:     "list of published repositories",
+		Long: `
+Display list of currently published snapshots.
+
+Example:
+
+    $ aptly publish list
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-list", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/publish_snapshot.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"strings"
+)
+
+func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 || len(args) > 2 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	var prefix string
+	if len(args) == 2 {
+		prefix = args[1]
+	} else {
+		prefix = ""
+	}
+
+	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	var sourceRepo *debian.RemoteRepo
+
+	if snapshot.SourceKind == "repo" && len(snapshot.SourceIDs) == 1 {
+		repoCollection := debian.NewRemoteRepoCollection(context.database)
+
+		sourceRepo, _ = repoCollection.ByUUID(snapshot.SourceIDs[0])
+	}
+
+	component := cmd.Flag.Lookup("component").Value.String()
+	if component == "" {
+		if sourceRepo != nil && len(sourceRepo.Components) == 1 {
+			component = sourceRepo.Components[0]
+		} else {
+			component = "main"
+		}
+	}
+
+	distribution := cmd.Flag.Lookup("distribution").Value.String()
+	if distribution == "" {
+		if sourceRepo != nil {
+			distribution = sourceRepo.Distribution
+		}
+
+		if distribution == "" {
+			return fmt.Errorf("unable to guess distribution name, please specify explicitly")
+		}
+	}
+
+	published, err := debian.NewPublishedRepo(prefix, distribution, component, context.architecturesList, snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	duplicate := publishedCollecton.CheckDuplicate(published)
+	if duplicate != nil {
+		publishedCollecton.LoadComplete(duplicate, snapshotCollection)
+		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
+	}
+
+	signer, err := getSigner(cmd)
+	if err != nil {
+		return fmt.Errorf("unable to initialize GPG signer: %s", err)
+	}
+
+	packageCollection := debian.NewPackageCollection(context.database)
+	err = published.Publish(context.packagePool, context.publishedStorage, packageCollection, signer, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to publish: %s", err)
+	}
+
+	err = publishedCollecton.Add(published)
+	if err != nil {
+		return fmt.Errorf("unable to save to DB: %s", err)
+	}
+
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
+		prefix += "/"
+	}
+
+	context.progress.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
+		snapshot.Name, context.publishedStorage.PublicPath())
+	context.progress.Printf("Now you can add following line to apt sources:\n")
+	context.progress.Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
+	if utils.StrSliceHasItem(published.Architectures, "source") {
+		context.progress.Printf("  deb-src http://your-server/%s %s %s\n", prefix, distribution, component)
+	}
+	context.progress.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
+	context.progress.Printf("\nYou can also use `aptly serve` to publish your repositories over HTTP quickly.\n")
+
+	return err
+}
+
+func makeCmdPublishSnapshot() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPublishSnapshot,
+		UsageLine: "snapshot <name> [<prefix>]",
+		Short:     "publish snapshot",
+		Long: `
+Command publish publishes snapshot as Debian repository ready to be consumed
+by apt tools. Published repostiories appear under rootDir/public directory.
+Valid GPG key is required for publishing.
+
+Example:
+
+    $ aptly publish snapshot wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
+	}
+	cmd.Flag.String("distribution", "", "distribution name to publish")
+	cmd.Flag.String("component", "", "component name to publish")
+	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.String("keyring", "", "GPG keyring to use (instead of default)")
+	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
+
+	return cmd
+}

cmd/repo.go

@@ -0,0 +1,25 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+)
+
+func makeCmdRepo() *commander.Command {
+	return &commander.Command{
+		UsageLine: "repo",
+		Short:     "manage local package repositories",
+		Subcommands: []*commander.Command{
+			makeCmdRepoAdd(),
+			makeCmdRepoCopy(),
+			makeCmdRepoCreate(),
+			makeCmdRepoDrop(),
+			makeCmdRepoImport(),
+			makeCmdRepoList(),
+			makeCmdRepoMove(),
+			makeCmdRepoRemove(),
+			makeCmdRepoShow(),
+		},
+		Flag: *flag.NewFlagSet("aptly-repo", flag.ExitOnError),
+	}
+}

cmd/repo_add.go

@@ -0,0 +1,206 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+func aptlyRepoAdd(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	verifier := &utils.GpgVerifier{}
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := localRepoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	err = localRepoCollection.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add: %s", err)
+	}
+
+	context.progress.Printf("Loading packages...\n")
+
+	packageCollection := debian.NewPackageCollection(context.database)
+	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	packageFiles := []string{}
+
+	for _, location := range args[1:] {
+		info, err := os.Stat(location)
+		if err != nil {
+			context.progress.ColoredPrintf("@y[!]@| @!Unable to process %s: %s@|", location, err)
+			continue
+		}
+		if info.IsDir() {
+			err = filepath.Walk(location, func(path string, info os.FileInfo, err error) error {
+				if err != nil {
+					return err
+				}
+				if info.IsDir() {
+					return nil
+				}
+
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+					packageFiles = append(packageFiles, path)
+				}
+
+				return nil
+			})
+		} else {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+				packageFiles = append(packageFiles, location)
+			} else {
+				context.progress.ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
+				continue
+			}
+		}
+	}
+
+	processedFiles := []string{}
+	sort.Strings(packageFiles)
+
+	for _, file := range packageFiles {
+		var (
+			stanza debian.Stanza
+			err    error
+			p      *debian.Package
+		)
+
+		candidateProcessedFiles := []string{}
+		isSourcePackage := strings.HasSuffix(file, ".dsc")
+
+		if isSourcePackage {
+			stanza, err = debian.GetControlFileFromDsc(file, verifier)
+
+			if err == nil {
+				stanza["Package"] = stanza["Source"]
+				delete(stanza, "Source")
+
+				p, err = debian.NewSourcePackageFromControlFile(stanza)
+			}
+		} else {
+			stanza, err = debian.GetControlFileFromDeb(file)
+			p = debian.NewPackageFromControlFile(stanza)
+		}
+		if err != nil {
+			context.progress.ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
+			continue
+		}
+
+		checksums, err := utils.ChecksumsForFile(file)
+		if err != nil {
+			return err
+		}
+
+		if isSourcePackage {
+			p.UpdateFiles(append(p.Files(), debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}))
+		} else {
+			p.UpdateFiles([]debian.PackageFile{debian.PackageFile{Filename: filepath.Base(file), Checksums: checksums}})
+		}
+
+		err = context.packagePool.Import(file, checksums.MD5)
+		if err != nil {
+			context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", file, err)
+			continue
+		}
+
+		candidateProcessedFiles = append(candidateProcessedFiles, file)
+
+		// go over all files, except for the last one (.dsc/.deb itself)
+		for _, f := range p.Files() {
+			if filepath.Base(f.Filename) == filepath.Base(file) {
+				continue
+			}
+			sourceFile := filepath.Join(filepath.Dir(file), filepath.Base(f.Filename))
+			err = context.packagePool.Import(sourceFile, f.Checksums.MD5)
+			if err != nil {
+				context.progress.ColoredPrintf("@y[!]@| @!Unable to import file %s into pool: %s@|", sourceFile, err)
+				break
+			}
+
+			candidateProcessedFiles = append(candidateProcessedFiles, sourceFile)
+		}
+		if err != nil {
+			// some files haven't been imported
+			continue
+		}
+
+		err = packageCollection.Update(p)
+		if err != nil {
+			context.progress.ColoredPrintf("@y[!]@| @!Unable to save package %s: %s@|", p, err)
+			continue
+		}
+
+		err = list.Add(p)
+		if err != nil {
+			context.progress.ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
+			continue
+		}
+
+		context.progress.ColoredPrintf("@g[+]@| %s added@|", p)
+		processedFiles = append(processedFiles, candidateProcessedFiles...)
+	}
+
+	repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+
+	err = localRepoCollection.Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to save: %s", err)
+	}
+
+	if cmd.Flag.Lookup("remove-files").Value.Get().(bool) {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err := os.Remove(file)
+			if err != nil {
+				return fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	return err
+}
+
+func makeCmdRepoAdd() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoAdd,
+		UsageLine: "add <name> <package file.deb>|<directory> ...",
+		Short:     "add packages to local repository",
+		Long: `
+Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+patterns. Every file discovered would be analyzed to extract metadata, package would be created and added
+to database. Files would be imported to internal package pool. For source packages, all required files are
+added as well automatically. Extra files for source package should be in the same directory as *.dsc file.
+
+Example:
+
+  $ aptly repo add testing myapp-0.1.2.deb incoming/
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+
+	return cmd
+}

cmd/repo_copy.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+)
+
+func makeCmdRepoCopy() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "copy <src-name> <dst-name> <package-spec> ...",
+		Short:     "copy packages between local repositories",
+		Long: `
+Command copy copies packages matching <package-spec> from local repo
+<src-name> to local repo <dst-name>.
+
+Example:
+
+  $ aptly repo copy testing stable 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-copy", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't copy, just show what would be copied")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_create.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyRepoCreate(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	repo := debian.NewLocalRepo(args[0], cmd.Flag.Lookup("comment").Value.String())
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+
+	err = localRepoCollection.Add(repo)
+	if err != nil {
+		return fmt.Errorf("unable to add local repo: %s", err)
+	}
+
+	fmt.Printf("\nLocal repo %s successfully added.\nYou can run 'aptly repo add %s ...' to add packages to repository.\n", repo, repo.Name)
+	return err
+}
+
+func makeCmdRepoCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoCreate,
+		UsageLine: "create <name>",
+		Short:     "create local repository",
+		Long: `
+Create local package repository. Repository would be empty when
+created, packages could be added from files, copied or moved from
+another local repository or imported from the mirror.
+
+Example:
+
+  $ aptly repo create testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-create", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("comment", "", "any text that would be used to described local repository")
+
+	return cmd
+}

cmd/repo_drop.go

@@ -0,0 +1,69 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyRepoDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := localRepoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshotCollection := debian.NewSnapshotCollection(context.database)
+		snapshots := snapshotCollection.ByLocalRepoSource(repo)
+
+		if len(snapshots) > 0 {
+			fmt.Printf("Local repo `%s` was used to create following snapshots:\n", repo.Name)
+			for _, snapshot := range snapshots {
+				fmt.Printf(" * %s\n", snapshot)
+			}
+
+			return fmt.Errorf("won't delete local repo with snapshots, use -force to override")
+		}
+	}
+
+	err = localRepoCollection.Drop(repo)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Local repo `%s` has been removed.\n", repo.Name)
+
+	return err
+}
+
+func makeCmdRepoDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete local repository",
+		Long: `
+Drop deletes information about local repo. Package data is not deleted
+(it could be still used by other mirrors or snapshots).
+
+Example:
+
+  $ aptly repo drop local-repo
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "force local repo deletion even if used by snapshots")
+
+	return cmd
+}

cmd/repo_import.go

@@ -0,0 +1,28 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+)
+
+func makeCmdRepoImport() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "import <src-mirror> <dst-repo> <package-spec> ...",
+		Short:     "import packages from mirror to local repository",
+		Long: `
+Command import looks up packages matching <package-spec> in mirror <src-mirror>
+and copies them to local repo <dst-repo>.
+
+Example:
+
+  $ aptly repo import wheezy-main testing nginx
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-import", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't import, just show what would be imported")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_list.go

@@ -0,0 +1,63 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlyRepoList(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 0 {
+		cmd.Usage()
+		return err
+	}
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+
+	if localRepoCollection.Len() > 0 {
+		fmt.Printf("List of mirrors:\n")
+		repos := make([]string, localRepoCollection.Len())
+		i := 0
+		localRepoCollection.ForEach(func(repo *debian.LocalRepo) error {
+			err := localRepoCollection.LoadComplete(repo)
+			if err != nil {
+				return err
+			}
+
+			repos[i] = fmt.Sprintf(" * %s (packages: %d)", repo.String(), repo.NumPackages())
+			i++
+			return nil
+		})
+
+		sort.Strings(repos)
+		for _, repo := range repos {
+			fmt.Println(repo)
+		}
+
+		fmt.Printf("\nTo get more information about local repository, run `aptly repo show <name>`.\n")
+	} else {
+		fmt.Printf("No local repositories found, create one with `aptly repo create ...`.\n")
+	}
+	return err
+}
+
+func makeCmdRepoList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoList,
+		UsageLine: "list",
+		Short:     "list local repositories",
+		Long: `
+List shows full list of local package repositories.
+
+Example:
+
+  $ aptly repo list
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-list", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/repo_move.go

@@ -0,0 +1,185 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return err
+	}
+
+	command := cmd.Name()
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+
+	dstRepo, err := localRepoCollection.ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	err = localRepoCollection.LoadComplete(dstRepo)
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	var (
+		srcRefList *debian.PackageRefList
+		srcRepo    *debian.LocalRepo
+	)
+
+	if command == "copy" || command == "move" {
+		srcRepo, err = localRepoCollection.ByName(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		if srcRepo.UUID == dstRepo.UUID {
+			return fmt.Errorf("unable to %s: source and destination are the same", command)
+		}
+
+		err = localRepoCollection.LoadComplete(srcRepo)
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		srcRefList = srcRepo.RefList()
+	} else if command == "import" {
+		repoCollection := debian.NewRemoteRepoCollection(context.database)
+
+		srcRepo, err := repoCollection.ByName(args[0])
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		err = repoCollection.LoadComplete(srcRepo)
+		if err != nil {
+			return fmt.Errorf("unable to %s: %s", command, err)
+		}
+
+		if srcRepo.RefList() == nil {
+			return fmt.Errorf("unable to %s: mirror not updated", command)
+		}
+
+		srcRefList = srcRepo.RefList()
+	} else {
+		panic("unexpected command")
+	}
+
+	context.progress.Printf("Loading packages...\n")
+
+	packageCollection := debian.NewPackageCollection(context.database)
+	dstList, err := debian.NewPackageListFromRefList(dstRepo.RefList(), packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	srcList, err := debian.NewPackageListFromRefList(srcRefList, packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	srcList.PrepareIndex()
+
+	var architecturesList []string
+
+	withDeps := cmd.Flag.Lookup("with-deps").Value.Get().(bool)
+
+	if withDeps {
+		dstList.PrepareIndex()
+
+		// Calculate architectures
+		if len(context.architecturesList) > 0 {
+			architecturesList = context.architecturesList
+		} else {
+			architecturesList = dstList.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	toProcess, err := srcList.Filter(args[2:], withDeps, dstList, context.dependencyOptions, architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	var verb string
+
+	if command == "move" {
+		verb = "moved"
+	} else if command == "copy" {
+		verb = "copied"
+	} else if command == "import" {
+		verb = "imported"
+	}
+
+	err = toProcess.ForEach(func(p *debian.Package) error {
+		err = dstList.Add(p)
+		if err != nil {
+			return err
+		}
+
+		if command == "move" {
+			srcList.Remove(p)
+		}
+		context.progress.ColoredPrintf("@g[o]@| %s %s", p, verb)
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("unable to %s: %s", command, err)
+	}
+
+	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	} else {
+		dstRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(dstList))
+
+		err = localRepoCollection.Update(dstRepo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+
+		if command == "move" {
+			srcRepo.UpdateRefList(debian.NewPackageRefListFromPackageList(srcList))
+
+			err = localRepoCollection.Update(srcRepo)
+			if err != nil {
+				return fmt.Errorf("unable to save: %s", err)
+			}
+		}
+	}
+
+	return err
+}
+
+func makeCmdRepoMove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoMoveCopyImport,
+		UsageLine: "move <src-name> <dst-name> <package-spec> ...",
+		Short:     "move packages between local repositories",
+		Long: `
+Command move moves packages matching <package-spec> from local repo
+<src-name> to local repo <dst-name>.
+
+Example:
+
+  $ aptly repo move testing stable 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-move", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't move, just show what would be moved")
+	cmd.Flag.Bool("with-deps", false, "follow dependencies when processing package-spec")
+
+	return cmd
+}

cmd/repo_remove.go

@@ -0,0 +1,85 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyRepoRemove(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := localRepoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	err = localRepoCollection.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	context.progress.Printf("Loading packages...\n")
+
+	packageCollection := debian.NewPackageCollection(context.database)
+	list, err := debian.NewPackageListFromRefList(repo.RefList(), packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	list.PrepareIndex()
+	toRemove, err := list.Filter(args[1:], false, nil, 0, nil)
+	if err != nil {
+		return fmt.Errorf("unable to remove: %s", err)
+	}
+
+	toRemove.ForEach(func(p *debian.Package) error {
+		list.Remove(p)
+		context.progress.ColoredPrintf("@r[-]@| %s removed", p)
+		return nil
+	})
+
+	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+		context.progress.Printf("\nChanges not saved, as dry run has been requested.\n")
+	} else {
+		repo.UpdateRefList(debian.NewPackageRefListFromPackageList(list))
+
+		err = localRepoCollection.Update(repo)
+		if err != nil {
+			return fmt.Errorf("unable to save: %s", err)
+		}
+	}
+
+	return err
+}
+
+func makeCmdRepoRemove() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoRemove,
+		UsageLine: "remove <name> <package-spec> ...",
+		Short:     "remove packages from local repository",
+		Long: `
+Commands removes packages matching <package-spec> from local repository
+<name>. If removed packages are not referenced by other repos or
+snapshots, they can be removed completely (including files) by running
+'aptly db cleanup'.
+
+Example:
+
+  $ aptly repo remove testing 'myapp (=0.1.12)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-add", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't remove, just show what would be removed")
+
+	return cmd
+}

cmd/repo_show.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlyRepoShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	localRepoCollection := debian.NewLocalRepoCollection(context.database)
+	repo, err := localRepoCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = localRepoCollection.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", repo.Name)
+	fmt.Printf("Comment: %s\n", repo.Comment)
+	fmt.Printf("Number of packages: %d\n", repo.NumPackages())
+
+	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		ListPackagesRefList(repo.RefList())
+	}
+
+	return err
+}
+
+func makeCmdRepoShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyRepoShow,
+		UsageLine: "show <name>",
+		Short:     "show details about local repository",
+		Long: `
+Show shows full information about local package repository.
+
+ex:
+  $ aptly repo show testing
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-packages", false, "show list of packages")
+
+	return cmd
+}

cmd/serve.go

@@ -0,0 +1,113 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/utils"
+	"net"
+	"net/http"
+	"os"
+	"sort"
+)
+
+func aptlyServe(cmd *commander.Command, args []string) error {
+	var err error
+
+	publishedCollection := debian.NewPublishedRepoCollection(context.database)
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	if publishedCollection.Len() == 0 {
+		fmt.Printf("No published repositories, unable to serve.\n")
+		return nil
+	}
+
+	listen := cmd.Flag.Lookup("listen").Value.String()
+
+	listenHost, listenPort, err := net.SplitHostPort(listen)
+
+	if err != nil {
+		return fmt.Errorf("wrong -listen specification: %s", err)
+	}
+
+	if listenHost == "" {
+		listenHost, err = os.Hostname()
+		if err != nil {
+			listenHost = "localhost"
+		}
+	}
+
+	fmt.Printf("Serving published repositories, recommended apt sources list:\n\n")
+
+	sources := make(sort.StringSlice, 0, publishedCollection.Len())
+	published := make(map[string]*debian.PublishedRepo, publishedCollection.Len())
+
+	err = publishedCollection.ForEach(func(repo *debian.PublishedRepo) error {
+		err := publishedCollection.LoadComplete(repo, snapshotCollection)
+		if err != nil {
+			return err
+		}
+
+		sources = append(sources, repo.String())
+		published[repo.String()] = repo
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+
+	sort.Strings(sources)
+
+	for _, source := range sources {
+		repo := published[source]
+
+		prefix := repo.Prefix
+		if prefix == "." {
+			prefix = ""
+		} else {
+			prefix += "/"
+		}
+
+		fmt.Printf("# %s\ndeb http://%s:%s/%s %s %s\n",
+			repo, listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+
+		if utils.StrSliceHasItem(repo.Architectures, "source") {
+			fmt.Printf("deb-src http://%s:%s/%s %s %s\n",
+				listenHost, listenPort, prefix, repo.Distribution, repo.Component)
+		}
+	}
+
+	context.database.Close()
+
+	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
+
+	err = http.ListenAndServe(listen, http.FileServer(http.Dir(context.publishedStorage.PublicPath())))
+	if err != nil {
+		return fmt.Errorf("unable to serve: %s", err)
+	}
+	return nil
+}
+
+func makeCmdServe() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyServe,
+		UsageLine: "serve",
+		Short:     "HTTP serve published repositories",
+		Long: `
+Command serve starts embedded HTTP server (not suitable for real production usage) to serve
+contents of public/ subdirectory of aptly's root that contains published repositories.
+
+Example:
+
+  $ aptly serve -listen=:8080
+`,
+		Flag: *flag.NewFlagSet("aptly-serve", flag.ExitOnError),
+	}
+
+	cmd.Flag.String("listen", ":8080", "host:port for HTTP listening")
+
+	return cmd
+}

cmd/snapshot.go

@@ -0,0 +1,24 @@
+package cmd
+
+import (
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+)
+
+func makeCmdSnapshot() *commander.Command {
+	return &commander.Command{
+		UsageLine: "snapshot",
+		Short:     "manage snapshots of repositories",
+		Subcommands: []*commander.Command{
+			makeCmdSnapshotCreate(),
+			makeCmdSnapshotList(),
+			makeCmdSnapshotShow(),
+			makeCmdSnapshotVerify(),
+			makeCmdSnapshotPull(),
+			makeCmdSnapshotDiff(),
+			makeCmdSnapshotMerge(),
+			makeCmdSnapshotDrop(),
+		},
+		Flag: *flag.NewFlagSet("aptly-snapshot", flag.ExitOnError),
+	}
+}

cmd/snapshot_create.go

@@ -0,0 +1,105 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
+	var (
+		err      error
+		snapshot *debian.Snapshot
+	)
+
+	if len(args) == 4 && args[1] == "from" && args[2] == "mirror" {
+		// aptly snapshot create snap from mirror mirror
+		repoName, snapshotName := args[3], args[0]
+
+		repoCollection := debian.NewRemoteRepoCollection(context.database)
+		repo, err := repoCollection.ByName(repoName)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = repoCollection.LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = debian.NewSnapshotFromRepository(snapshotName, repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+	} else if len(args) == 4 && args[1] == "from" && args[2] == "repo" {
+		// aptly snapshot create snap from repo repo
+		localRepoName, snapshotName := args[3], args[0]
+
+		localRepoCollection := debian.NewLocalRepoCollection(context.database)
+		repo, err := localRepoCollection.ByName(localRepoName)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		err = localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		snapshot, err = debian.NewSnapshotFromLocalRepo(snapshotName, repo)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+	} else if len(args) == 2 && args[1] == "empty" {
+		// aptly snapshot create snap empty
+		snapshotName := args[0]
+
+		packageList := debian.NewPackageList()
+
+		snapshot = debian.NewSnapshotFromPackageList(snapshotName, nil, packageList, "Created as empty")
+	} else {
+		cmd.Usage()
+		return err
+	}
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	err = snapshotCollection.Add(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to add snapshot: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", snapshot.Name, snapshot.Name)
+
+	return err
+}
+
+func makeCmdSnapshotCreate() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotCreate,
+		UsageLine: "create <name> from mirror <mirror-name> | from repo <repo-name> | empty",
+		Short:     "creates snapshot of mirror (local repository) contents",
+		Long: `
+Command create <name> from mirror makes persistent immutable snapshot of remote
+repository mirror. Snapshot could be published or further modified using
+merge, pull and other aptly features.
+
+Command create <name> from repo makes persistent immutable snapshot of local
+repository. Snapshot could be processed as mirror snapshots, and mixed with
+snapshots of remote mirrors.
+
+Command create <name> empty creates empty snapshot that could be used as a
+basis for snapshot pull operations, for example. As snapshots are immutable,
+creating one empty snapshot should be enough.
+
+Example:
+
+  $ aptly snapshot create wheezy-main-today from mirror wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-create", flag.ExitOnError),
+	}
+
+	return cmd
+
+}

cmd/snapshot_diff.go

@@ -0,0 +1,115 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return err
+	}
+
+	onlyMatching := cmd.Flag.Lookup("only-matching").Value.Get().(bool)
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	packageCollection := debian.NewPackageCollection(context.database)
+
+	// Load <name-a> snapshot
+	snapshotA, err := snapshotCollection.ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot A: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(snapshotA)
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot A: %s", err)
+	}
+
+	// Load <name-b> snapshot
+	snapshotB, err := snapshotCollection.ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot B: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(snapshotB)
+	if err != nil {
+		return fmt.Errorf("unable to load snapshot B: %s", err)
+	}
+
+	// Calculate diff
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), packageCollection)
+	if err != nil {
+		return fmt.Errorf("unable to calculate diff: %s", err)
+	}
+
+	if len(diff) == 0 {
+		context.progress.Printf("Snapshots are identical.\n")
+	} else {
+		context.progress.Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
+		for _, pdiff := range diff {
+			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
+				continue
+			}
+
+			var verA, verB, pkg, arch, code string
+
+			if pdiff.Left == nil {
+				verA = "-"
+				verB = pdiff.Right.Version
+				pkg = pdiff.Right.Name
+				arch = pdiff.Right.Architecture
+			} else {
+				pkg = pdiff.Left.Name
+				arch = pdiff.Left.Architecture
+				verA = pdiff.Left.Version
+				if pdiff.Right == nil {
+					verB = "-"
+				} else {
+					verB = pdiff.Right.Version
+				}
+			}
+
+			if pdiff.Left == nil {
+				code = "@g+@|"
+			} else {
+				if pdiff.Right == nil {
+					code = "@r-@|"
+				} else {
+					code = "@y!@|"
+				}
+			}
+
+			context.progress.ColoredPrintf(code+" %-6s | %-40s | %-40s | %-40s", arch, pkg, verA, verB)
+		}
+	}
+
+	return err
+}
+
+func makeCmdSnapshotDiff() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotDiff,
+		UsageLine: "diff <name-a> <name-b>",
+		Short:     "difference between two snapshots",
+		Long: `
+Displays difference in packages between two snapshots. Snapshot is a list
+of packages, so difference between snapshots is a difference between package
+lists. Package could be either completely missing in one snapshot, or package
+is present in both snapshots with different versions.
+
+Example:
+
+    $ aptly snapshot diff -only-matching wheezy-main wheezy-backports
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-diff", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("only-matching", false, "display diff only for matching packages (don't display missing packages)")
+
+	return cmd
+}

cmd/snapshot_drop.go

@@ -0,0 +1,83 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlySnapshotDrop(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	publishedRepoCollection := debian.NewPublishedRepoCollection(context.database)
+	published := publishedRepoCollection.BySnapshot(snapshot)
+
+	if len(published) > 0 {
+		fmt.Printf("Snapshot `%s` is published currently:\n", snapshot.Name)
+		for _, repo := range published {
+			err = publishedRepoCollection.LoadComplete(repo, snapshotCollection)
+			if err != nil {
+				return fmt.Errorf("unable to load published: %s", err)
+			}
+			fmt.Printf(" * %s\n", repo)
+		}
+
+		return fmt.Errorf("unable to drop: snapshot is published")
+	}
+
+	force := cmd.Flag.Lookup("force").Value.Get().(bool)
+	if !force {
+		snapshots := snapshotCollection.BySnapshotSource(snapshot)
+		if len(snapshots) > 0 {
+			fmt.Printf("Snapshot `%s` was used as a source in following snapshots:\n", snapshot.Name)
+			for _, snap := range snapshots {
+				fmt.Printf(" * %s\n", snap)
+			}
+
+			return fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use -force to override")
+		}
+	}
+
+	err = snapshotCollection.Drop(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
+	fmt.Printf("Snapshot `%s` has been dropped.\n", snapshot.Name)
+
+	return err
+}
+
+func makeCmdSnapshotDrop() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotDrop,
+		UsageLine: "drop <name>",
+		Short:     "delete snapshot",
+		Long: `
+Drop removes information about snapshot. If snapshot is published,
+it can't be dropped.
+
+Example:
+
+    $ aptly snapshot drop wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-drop", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("force", false, "remove snapshot even if it was used as source for other snapshots")
+
+	return cmd
+}

cmd/snapshot_list.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlySnapshotList(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 0 {
+		cmd.Usage()
+		return err
+	}
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	if snapshotCollection.Len() > 0 {
+		fmt.Printf("List of snapshots:\n")
+
+		snapshots := make([]string, snapshotCollection.Len())
+
+		i := 0
+		snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
+			snapshots[i] = snapshot.String()
+			i++
+			return nil
+		})
+
+		sort.Strings(snapshots)
+		for _, snapshot := range snapshots {
+			fmt.Printf(" * %s\n", snapshot)
+		}
+
+		fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
+	} else {
+		fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
+	}
+	return err
+
+}
+
+func makeCmdSnapshotList() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotList,
+		UsageLine: "list",
+		Short:     "list snapshots",
+		Long: `
+Command list shows full list of snapshots created.
+
+Example:
+
+  $ aptly snapshot list
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-list", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/snapshot_merge.go

@@ -0,0 +1,79 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"strings"
+)
+
+func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 2 {
+		cmd.Usage()
+		return err
+	}
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+
+	sources := make([]*debian.Snapshot, len(args)-1)
+
+	for i := 0; i < len(args)-1; i++ {
+		sources[i], err = snapshotCollection.ByName(args[i+1])
+		if err != nil {
+			return fmt.Errorf("unable to load snapshot: %s", err)
+		}
+
+		err = snapshotCollection.LoadComplete(sources[i])
+		if err != nil {
+			return fmt.Errorf("unable to load snapshot: %s", err)
+		}
+	}
+
+	result := sources[0].RefList()
+
+	for i := 1; i < len(sources); i++ {
+		result = result.Merge(sources[i].RefList(), true)
+	}
+
+	sourceDescription := make([]string, len(sources))
+	for i, s := range sources {
+		sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
+	}
+
+	// Create <destination> snapshot
+	destination := debian.NewSnapshotFromRefList(args[0], sources, result,
+		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
+
+	err = snapshotCollection.Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotMerge() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMerge,
+		UsageLine: "merge <destination> <source> [<source>...]",
+		Short:     "merges snapshots",
+		Long: `
+Merge merges several <source> snapshots into one <destination> snapshot.
+Merge happens from left to right. Packages with the same name-architecture
+pair are replaced during merge (package from latest snapshot on the list
+wins). If run with only one source snapshot, merge copies <source> into
+<destination>.
+
+Example:
+
+    $ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-merge", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/snapshot_pull.go

@@ -0,0 +1,193 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+	"strings"
+)
+
+func aptlySnapshotPull(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 4 {
+		cmd.Usage()
+		return err
+	}
+
+	noDeps := cmd.Flag.Lookup("no-deps").Value.Get().(bool)
+	noRemove := cmd.Flag.Lookup("no-remove").Value.Get().(bool)
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	packageCollection := debian.NewPackageCollection(context.database)
+
+	// Load <name> snapshot
+	snapshot, err := snapshotCollection.ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	// Load <source> snapshot
+	source, err := snapshotCollection.ByName(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to pull: %s", err)
+	}
+
+	context.progress.Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
+		snapshot, source, args[2])
+
+	// Convert snapshot to package list
+	context.progress.Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
+	packageList, err := debian.NewPackageListFromRefList(snapshot.RefList(), packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	sourcePackageList, err := debian.NewPackageListFromRefList(source.RefList(), packageCollection, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.progress.Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+	sourcePackageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.architecturesList) > 0 {
+		architecturesList = context.architecturesList
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial dependencies out of arguments
+	initialDependencies := make([]debian.Dependency, len(args)-3)
+	for i, arg := range args[3:] {
+		initialDependencies[i], err = debian.ParseDependency(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse argument: %s", err)
+		}
+	}
+
+	// Perform pull
+	for _, arch := range architecturesList {
+		dependencies := make([]debian.Dependency, len(initialDependencies), 128)
+		for i := range dependencies {
+			dependencies[i] = initialDependencies[i]
+			dependencies[i].Architecture = arch
+		}
+
+		// Go over list of initial dependencies + list of dependencies found
+		for i := 0; i < len(dependencies); i++ {
+			dep := dependencies[i]
+
+			// Search for package that can satisfy dependencies
+			pkg := sourcePackageList.Search(dep)
+			if pkg == nil {
+				context.progress.ColoredPrintf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
+				continue
+			}
+
+			if !noRemove {
+				// Remove all packages with the same name and architecture
+				for p := packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}); p != nil; {
+					packageList.Remove(p)
+					context.progress.ColoredPrintf("@r[-]@| %s removed", p)
+					p = packageList.Search(debian.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name})
+				}
+			}
+
+			// Add new discovered package
+			packageList.Add(pkg)
+			context.progress.ColoredPrintf("@g[+]@| %s added", pkg)
+
+			if noDeps {
+				continue
+			}
+
+			// Find missing dependencies for single added package
+			pL := debian.NewPackageList()
+			pL.Add(pkg)
+
+			missing, err := pL.VerifyDependencies(context.dependencyOptions, []string{arch}, packageList, nil)
+			if err != nil {
+				context.progress.ColoredPrintf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
+			}
+
+			// Append missing dependencies to the list of dependencies to satisfy
+			for _, misDep := range missing {
+				found := false
+				for _, d := range dependencies {
+					if d == misDep {
+						found = true
+						break
+					}
+				}
+
+				if !found {
+					dependencies = append(dependencies, misDep)
+				}
+			}
+		}
+	}
+
+	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
+		context.progress.Printf("\nNot creating snapshot, as dry run was requested.\n")
+	} else {
+		// Create <destination> snapshot
+		destination := debian.NewSnapshotFromPackageList(args[2], []*debian.Snapshot{snapshot, source}, packageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
+
+		err = snapshotCollection.Add(destination)
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
+		context.progress.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+	}
+	return err
+}
+
+func makeCmdSnapshotPull() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotPull,
+		UsageLine: "pull <name> <source> <destination> <package-name> ...",
+		Short:     "pull packages from another snapshot",
+		Long: `
+Command pull pulls new packages along with its dependencies to snapshot <name>
+from snapshot <source>. Pull can upgrade package version in <name> with
+versions from <source> following dependencies. New snapshot <destination>
+is created as result of this process. Packages could be specified simply
+as 'package-name' or as dependency 'package-name (>= version)'.
+
+Example:
+
+    $ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-pull", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
+	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
+	cmd.Flag.Bool("no-remove", false, "don't remove other package versions when pulling package")
+
+	return cmd
+}

cmd/snapshot_show.go

@@ -0,0 +1,61 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+)
+
+func aptlySnapshotShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return err
+	}
+
+	name := args[0]
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	snapshot, err := snapshotCollection.ByName(name)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	err = snapshotCollection.LoadComplete(snapshot)
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	fmt.Printf("Name: %s\n", snapshot.Name)
+	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
+	fmt.Printf("Description: %s\n", snapshot.Description)
+	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
+
+	withPackages := cmd.Flag.Lookup("with-packages").Value.Get().(bool)
+	if withPackages {
+		ListPackagesRefList(snapshot.RefList())
+	}
+
+	return err
+}
+
+func makeCmdSnapshotShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotShow,
+		UsageLine: "show <name>",
+		Short:     "shows details about snapshot",
+		Long: `
+Command show displays full information about snapshot.
+
+Example:
+
+    $ aptly snapshot show wheezy-main
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-packages", false, "show list of packages")
+
+	return cmd
+}

cmd/snapshot_verify.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/debian"
+	"sort"
+)
+
+func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 1 {
+		cmd.Usage()
+		return err
+	}
+
+	snapshotCollection := debian.NewSnapshotCollection(context.database)
+	packageCollection := debian.NewPackageCollection(context.database)
+
+	snapshots := make([]*debian.Snapshot, len(args))
+	for i := range snapshots {
+		snapshots[i], err = snapshotCollection.ByName(args[i])
+		if err != nil {
+			return fmt.Errorf("unable to verify: %s", err)
+		}
+
+		err = snapshotCollection.LoadComplete(snapshots[i])
+		if err != nil {
+			return fmt.Errorf("unable to verify: %s", err)
+		}
+	}
+
+	context.progress.Printf("Loading packages...\n")
+
+	packageList, err := debian.NewPackageListFromRefList(snapshots[0].RefList(), packageCollection, context.progress)
+	if err != nil {
+		fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	sourcePackageList := debian.NewPackageList()
+	err = sourcePackageList.Append(packageList)
+	if err != nil {
+		fmt.Errorf("unable to merge sources: %s", err)
+	}
+
+	for i := 1; i < len(snapshots); i++ {
+		pL, err := debian.NewPackageListFromRefList(snapshots[i].RefList(), packageCollection, context.progress)
+		if err != nil {
+			fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		err = sourcePackageList.Append(pL)
+		if err != nil {
+			fmt.Errorf("unable to merge sources: %s", err)
+		}
+	}
+
+	sourcePackageList.PrepareIndex()
+
+	var architecturesList []string
+
+	if len(context.architecturesList) > 0 {
+		architecturesList = context.architecturesList
+	} else {
+		architecturesList = packageList.Architectures(true)
+	}
+
+	if len(architecturesList) == 0 {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	context.progress.Printf("Verifying...\n")
+
+	missing, err := packageList.VerifyDependencies(context.dependencyOptions, architecturesList, sourcePackageList, context.progress)
+	if err != nil {
+		return fmt.Errorf("unable to verify dependencies: %s", err)
+	}
+
+	if len(missing) == 0 {
+		context.progress.Printf("All dependencies are satisfied.\n")
+	} else {
+		context.progress.Printf("Missing dependencies (%d):\n", len(missing))
+		deps := make([]string, len(missing))
+		i := 0
+		for _, dep := range missing {
+			deps[i] = dep.String()
+			i++
+		}
+
+		sort.Strings(deps)
+
+		for _, dep := range deps {
+			context.progress.Printf("  %s\n", dep)
+		}
+	}
+
+	return err
+}
+
+func makeCmdSnapshotVerify() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotVerify,
+		UsageLine: "verify <name> [<source> ...]",
+		Short:     "verify dependencies in snapshot",
+		Long: `
+Verify does depenency resolution in snapshot <name>, possibly using additional
+snapshots <source> as dependency sources. All unsatisfied dependencies are
+printed.
+
+Example:
+
+    $ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-verify", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/version.go

@@ -1,13 +1,14 @@
-package main
+package cmd
 
 import (
 	"fmt"
 	"github.com/gonuts/commander"
 	"github.com/gonuts/flag"
+	"github.com/smira/aptly/aptly"
 )
 
 func aptlyVersion(cmd *commander.Command, args []string) error {
-	fmt.Printf("aptly version: %s\n", Version)
+	fmt.Printf("aptly version: %s\n", aptly.Version)
 	return nil
 }
 

cmd_mirror.go

@@ -1,235 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
-	"github.com/smira/aptly/utils"
-	"sort"
-	"strings"
-)
-
-func aptlyMirrorList(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 0 {
-		cmd.Usage()
-		return err
-	}
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	if repoCollection.Len() > 0 {
-		fmt.Printf("List of mirrors:\n")
-		repos := make(sort.StringSlice, repoCollection.Len())
-		i := 0
-		repoCollection.ForEach(func(repo *debian.RemoteRepo) error {
-			repos[i] = repo.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(repos)
-		for _, repo := range repos {
-			fmt.Printf(" * %s\n", repo)
-		}
-
-		fmt.Printf("\nTo get more information about mirror, run `aptly mirror show <name>`.\n")
-	} else {
-		fmt.Printf("No mirrors found, create one with `aptly mirror create ...`.\n")
-	}
-	return err
-}
-
-func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 3 {
-		cmd.Usage()
-		return err
-	}
-
-	repo, err := debian.NewRemoteRepo(args[0], args[1], args[2], args[3:], context.architecturesList)
-	if err != nil {
-		return fmt.Errorf("unable to create mirror: %s", err)
-	}
-
-	err = repo.Fetch(context.downloader)
-	if err != nil {
-		return fmt.Errorf("unable to fetch mirror: %s", err)
-	}
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-	err = repoCollection.Add(repo)
-	if err != nil {
-		return fmt.Errorf("unable to add mirror: %s", err)
-	}
-
-	fmt.Printf("\nMirror %s successfully added.\nYou can run 'aptly mirror update %s' to download repository contents.\n", repo, repo.Name)
-	return err
-}
-
-func aptlyMirrorShow(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 1 {
-		cmd.Usage()
-		return err
-	}
-
-	name := args[0]
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to show: %s", err)
-	}
-
-	err = repoCollection.LoadComplete(repo)
-	if err != nil {
-		return fmt.Errorf("unable to show: %s", err)
-	}
-
-	fmt.Printf("Name: %s\n", repo.Name)
-	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
-	fmt.Printf("Distribution: %s\n", repo.Distribution)
-	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
-	fmt.Printf("Architectures: %s\n", strings.Join(repo.Architectures, ", "))
-	if repo.LastDownloadDate.IsZero() {
-		fmt.Printf("Last update: never\n")
-	} else {
-		fmt.Printf("Last update: %s\n", repo.LastDownloadDate.Format("2006-01-02 15:04:05 MST"))
-		fmt.Printf("Number of packages: %d\n", repo.NumPackages())
-	}
-
-	fmt.Printf("\nInformation from release file:\n")
-	for _, k := range utils.StrMapSortedKeys(repo.Meta) {
-		fmt.Printf("%s: %s\n", k, repo.Meta[k])
-	}
-
-	return err
-}
-
-func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 1 {
-		cmd.Usage()
-		return err
-	}
-
-	name := args[0]
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	err = repoCollection.LoadComplete(repo)
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	err = repo.Fetch(context.downloader)
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	err = repo.Download(context.downloader, packageCollection, context.packageRepository)
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	err = repoCollection.Update(repo)
-	if err != nil {
-		return fmt.Errorf("unable to update: %s", err)
-	}
-
-	fmt.Printf("\nMirror `%s` has been successfully updated.\n", repo.Name)
-	return err
-}
-
-func makeCmdMirrorCreate() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyMirrorCreate,
-		UsageLine: "create <name> <archive url> <distribution> [<component1> ...]",
-		Short:     "create new mirror of Debian repository",
-		Long: `
-Create records information about new mirror and fetches Release file (it doesn't download packages).
-
-ex:
-  $ aptly mirror create wheezy-main http://mirror.yandex.ru/debian/ wheezy main
-`,
-		Flag: *flag.NewFlagSet("aptly-mirror-create", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdMirrorList() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyMirrorList,
-		UsageLine: "list",
-		Short:     "list mirrors of remote repositories",
-		Long: `
-List shows full list of remote repositories.
-
-ex:
-  $ aptly mirror list
-`,
-		Flag: *flag.NewFlagSet("aptly-mirror-list", flag.ExitOnError),
-	}
-	cmd.Flag.Bool("v", false, "enable verbose output")
-
-	return cmd
-}
-
-func makeCmdMirrorShow() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyMirrorShow,
-		UsageLine: "show <name>",
-		Short:     "show details about remote repository mirror",
-		Long: `
-Show shows full information about mirror.
-
-ex:
-  $ aptly mirror show wheezy-main
-`,
-		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdMirrorUpdate() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyMirrorUpdate,
-		UsageLine: "update <name>",
-		Short:     "update packages from remote mirror",
-		Long: `
-Update downloads list of packages and package files.
-
-ex:
-  $ aptly mirror update wheezy-main
-`,
-		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdMirror() *commander.Command {
-	return &commander.Command{
-		UsageLine: "mirror",
-		Short:     "manage mirrors of remote repositories",
-		Subcommands: []*commander.Command{
-			makeCmdMirrorCreate(),
-			makeCmdMirrorList(),
-			makeCmdMirrorShow(),
-			//makeCmdMirrorDestroy(),
-			makeCmdMirrorUpdate(),
-		},
-		Flag: *flag.NewFlagSet("aptly-mirror", flag.ExitOnError),
-	}
-}

cmd_publish.go

@@ -1,237 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
-	"github.com/smira/aptly/utils"
-	"sort"
-	"strings"
-)
-
-func aptlyPublishSnapshot(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 1 || len(args) > 2 {
-		cmd.Usage()
-		return err
-	}
-
-	name := args[0]
-
-	var prefix string
-	if len(args) == 2 {
-		prefix = args[1]
-	} else {
-		prefix = ""
-	}
-
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
-
-	var sourceRepo *debian.RemoteRepo
-
-	if snapshot.SourceKind == "repo" && len(snapshot.SourceIDs) == 1 {
-		repoCollection := debian.NewRemoteRepoCollection(context.database)
-
-		sourceRepo, _ = repoCollection.ByUUID(snapshot.SourceIDs[0])
-	}
-
-	component := cmd.Flag.Lookup("component").Value.String()
-	if component == "" {
-		if sourceRepo != nil && len(sourceRepo.Components) == 1 {
-			component = sourceRepo.Components[0]
-		} else {
-			component = "main"
-		}
-	}
-
-	distribution := cmd.Flag.Lookup("distribution").Value.String()
-	if distribution == "" {
-		if sourceRepo != nil {
-			distribution = sourceRepo.Distribution
-		} else {
-			return fmt.Errorf("unable to guess distribution name, please specify explicitly")
-		}
-	}
-
-	signer := &utils.GpgSigner{}
-	signer.SetKey(cmd.Flag.Lookup("gpg-key").Value.String())
-
-	published, err := debian.NewPublishedRepo(prefix, distribution, component, context.architecturesList, snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
-
-	duplicate := publishedCollecton.CheckDuplicate(published)
-	if duplicate != nil {
-		publishedCollecton.LoadComplete(duplicate, snapshotCollection)
-		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
-	}
-
-	packageCollection := debian.NewPackageCollection(context.database)
-	err = published.Publish(context.packageRepository, packageCollection, signer)
-	if err != nil {
-		return fmt.Errorf("unable to publish: %s", err)
-	}
-
-	err = publishedCollecton.Add(published)
-	if err != nil {
-		return fmt.Errorf("unable to save to DB: %s", err)
-	}
-
-	if prefix != "" && !strings.HasSuffix(prefix, "/") {
-		prefix += "/"
-	}
-
-	fmt.Printf("\nSnapshot %s has been successfully published.\nPlease setup your webserver to serve directory '%s' with autoindexing.\n",
-		snapshot.Name, context.packageRepository.PublicPath())
-	fmt.Printf("Now you can add following line to apt sources:\n")
-	fmt.Printf("  deb http://your-server/%s %s %s\n", prefix, distribution, component)
-	fmt.Printf("Don't forget to add your GPG key to apt with apt-key.\n")
-
-	return err
-}
-
-func aptlyPublishList(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 0 {
-		cmd.Usage()
-		return err
-	}
-
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	if publishedCollecton.Len() == 0 {
-		fmt.Printf("No snapshots have been published. Publish a snapshot by running `aptly publish snapshot ...`.\n")
-		return err
-	}
-
-	published := make(sort.StringSlice, 0, publishedCollecton.Len())
-
-	err = publishedCollecton.ForEach(func(repo *debian.PublishedRepo) error {
-		err := publishedCollecton.LoadComplete(repo, snapshotCollection)
-		if err != nil {
-			return err
-		}
-
-		published = append(published, repo.String())
-		return nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("unable to load list of repos: %s", err)
-	}
-
-	sort.Strings(published)
-
-	fmt.Printf("Published repositories:\n")
-
-	for _, description := range published {
-		fmt.Printf("  * %s\n", description)
-	}
-
-	return err
-}
-
-func aptlyPublishDrop(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 1 || len(args) > 2 {
-		cmd.Usage()
-		return err
-	}
-
-	distribution := args[0]
-	prefix := "."
-
-	if len(args) == 2 {
-		prefix = args[1]
-	}
-
-	publishedCollecton := debian.NewPublishedRepoCollection(context.database)
-
-	err = publishedCollecton.Remove(context.packageRepository, prefix, distribution)
-	if err != nil {
-		return fmt.Errorf("unable to remove: %s", err)
-	}
-
-	return err
-}
-
-func makeCmdPublishSnapshot() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyPublishSnapshot,
-		UsageLine: "snapshot <name> [<prefix>]",
-		Short:     "makes Debian repository out of snapshot",
-		Long: `
-Command publish oublishes snapshot as Debian repository ready to be used by apt tools.
-
-ex.
-	$ aptly publish snapshot wheezy-main
-`,
-		Flag: *flag.NewFlagSet("aptly-publish-snapshot", flag.ExitOnError),
-	}
-	cmd.Flag.String("distribution", "", "distribution name to publish")
-	cmd.Flag.String("component", "", "component name to publish")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
-
-	return cmd
-}
-
-func makeCmdPublishDrop() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyPublishDrop,
-		UsageLine: "drop <distribution> [<prefix>]",
-		Short:     "removes files of published repository",
-		Long: `
-Command removes whatever has been published under specified prefix and distribution name.
-
-ex.
-	$ aptly publish drop wheezy
-`,
-		Flag: *flag.NewFlagSet("aptly-publish-drop", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdPublishList() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlyPublishList,
-		UsageLine: "list",
-		Short:     "displays list of published repositories",
-		Long: `
-Display command displays list of currently published snapshots with information about published root.
-
-ex.
-	$ aptly publish list
-`,
-		Flag: *flag.NewFlagSet("aptly-publish-list", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdPublish() *commander.Command {
-	return &commander.Command{
-		UsageLine: "publish",
-		Short:     "manage published repositories",
-		Subcommands: []*commander.Command{
-			makeCmdPublishSnapshot(),
-			makeCmdPublishList(),
-			makeCmdPublishDrop(),
-		},
-		Flag: *flag.NewFlagSet("aptly-publish", flag.ExitOnError),
-	}
-}

cmd_snapshot.go

@@ -1,650 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/debian"
-	"github.com/wsxiaoys/terminal/color"
-	"sort"
-	"strings"
-)
-
-func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
-	var err error
-
-	if len(args) < 4 || args[1] != "from" || args[2] != "mirror" {
-		cmd.Usage()
-		return err
-	}
-
-	repoName, mirrorName := args[3], args[0]
-
-	repoCollection := debian.NewRemoteRepoCollection(context.database)
-	repo, err := repoCollection.ByName(repoName)
-	if err != nil {
-		return fmt.Errorf("unable to create snapshot: %s", err)
-	}
-
-	err = repoCollection.LoadComplete(repo)
-	if err != nil {
-		return fmt.Errorf("unable to create snapshot: %s", err)
-	}
-
-	snapshot, err := debian.NewSnapshotFromRepository(mirrorName, repo)
-	if err != nil {
-		return fmt.Errorf("unable to create snapshot: %s", err)
-	}
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	err = snapshotCollection.Add(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to add snapshot: %s", err)
-	}
-
-	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", snapshot.Name, snapshot.Name)
-
-	return err
-}
-
-func aptlySnapshotList(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 0 {
-		cmd.Usage()
-		return err
-	}
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	if snapshotCollection.Len() > 0 {
-		fmt.Printf("List of snapshots:\n")
-
-		snapshots := make(sort.StringSlice, snapshotCollection.Len())
-
-		i := 0
-		snapshotCollection.ForEach(func(snapshot *debian.Snapshot) error {
-			snapshots[i] = snapshot.String()
-			i++
-			return nil
-		})
-
-		sort.Strings(snapshots)
-		for _, snapshot := range snapshots {
-			fmt.Printf(" * %s\n", snapshot)
-		}
-
-		fmt.Printf("\nTo get more information about snapshot, run `aptly snapshot show <name>`.\n")
-	} else {
-		fmt.Printf("\nNo snapshots found, create one with `aptly snapshot create...`.\n")
-	}
-	return err
-
-}
-
-func aptlySnapshotShow(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 1 {
-		cmd.Usage()
-		return err
-	}
-
-	name := args[0]
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	snapshot, err := snapshotCollection.ByName(name)
-	if err != nil {
-		return fmt.Errorf("unable to show: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to show: %s", err)
-	}
-
-	fmt.Printf("Name: %s\n", snapshot.Name)
-	fmt.Printf("Created At: %s\n", snapshot.CreatedAt.Format("2006-01-02 15:04:05 MST"))
-	fmt.Printf("Description: %s\n", snapshot.Description)
-	fmt.Printf("Number of packages: %d\n", snapshot.NumPackages())
-	fmt.Printf("Packages:\n")
-
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	err = snapshot.RefList().ForEach(func(key []byte) error {
-		p, err := packageCollection.ByKey(key)
-		if err != nil {
-			return err
-		}
-		fmt.Printf("  %s\n", p)
-		return nil
-	})
-	if err != nil {
-		return fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	return err
-}
-
-func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 1 {
-		cmd.Usage()
-		return err
-	}
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	snapshots := make([]*debian.Snapshot, len(args))
-	for i := range snapshots {
-		snapshots[i], err = snapshotCollection.ByName(args[i])
-		if err != nil {
-			return fmt.Errorf("unable to verify: %s", err)
-		}
-
-		err = snapshotCollection.LoadComplete(snapshots[i])
-		if err != nil {
-			return fmt.Errorf("unable to verify: %s", err)
-		}
-	}
-
-	packageList, err := debian.NewPackageListFromRefList(snapshots[0].RefList(), packageCollection)
-	if err != nil {
-		fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	sourcePackageList := debian.NewPackageList()
-	err = sourcePackageList.Append(packageList)
-	if err != nil {
-		fmt.Errorf("unable to merge sources: %s", err)
-	}
-
-	for i := 1; i < len(snapshots); i++ {
-		pL, err := debian.NewPackageListFromRefList(snapshots[i].RefList(), packageCollection)
-		if err != nil {
-			fmt.Errorf("unable to load packages: %s", err)
-		}
-
-		err = sourcePackageList.Append(pL)
-		if err != nil {
-			fmt.Errorf("unable to merge sources: %s", err)
-		}
-	}
-
-	sourcePackageList.PrepareIndex()
-
-	var architecturesList []string
-
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
-	} else {
-		architecturesList = packageList.Architectures()
-	}
-
-	if len(architecturesList) == 0 {
-		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
-	}
-
-	missing, err := packageList.VerifyDependencies(context.dependencyOptions, architecturesList, sourcePackageList)
-	if err != nil {
-		return fmt.Errorf("unable to verify dependencies: %s", err)
-	}
-
-	if len(missing) == 0 {
-		fmt.Printf("All dependencies are satisfied.\n")
-	} else {
-		fmt.Printf("Missing dependencies (%d):\n", len(missing))
-		deps := make(sort.StringSlice, len(missing))
-		i := 0
-		for _, dep := range missing {
-			deps[i] = dep.String()
-			i++
-		}
-
-		sort.Strings(deps)
-
-		for _, dep := range deps {
-			fmt.Printf("  %s\n", dep)
-		}
-	}
-
-	return err
-}
-
-func aptlySnapshotPull(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 4 {
-		cmd.Usage()
-		return err
-	}
-
-	noDeps := cmd.Flag.Lookup("no-deps").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	// Load <name> snapshot
-	snapshot, err := snapshotCollection.ByName(args[0])
-	if err != nil {
-		return fmt.Errorf("unable to pull: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(snapshot)
-	if err != nil {
-		return fmt.Errorf("unable to pull: %s", err)
-	}
-
-	// Load <source> snapshot
-	source, err := snapshotCollection.ByName(args[1])
-	if err != nil {
-		return fmt.Errorf("unable to pull: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(source)
-	if err != nil {
-		return fmt.Errorf("unable to pull: %s", err)
-	}
-
-	fmt.Printf("Dependencies would be pulled into snapshot:\n    %s\nfrom snapshot:\n    %s\nand result would be saved as new snapshot %s.\n",
-		snapshot, source, args[2])
-
-	// Convert snapshot to package list
-	fmt.Printf("Loading packages (%d)...\n", snapshot.RefList().Len()+source.RefList().Len())
-	packageList, err := debian.NewPackageListFromRefList(snapshot.RefList(), packageCollection)
-	if err != nil {
-		return fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	sourcePackageList, err := debian.NewPackageListFromRefList(source.RefList(), packageCollection)
-	if err != nil {
-		return fmt.Errorf("unable to load packages: %s", err)
-	}
-
-	fmt.Printf("Building indexes...\n")
-	packageList.PrepareIndex()
-	sourcePackageList.PrepareIndex()
-
-	// Calculate architectures
-	var architecturesList []string
-
-	if len(context.architecturesList) > 0 {
-		architecturesList = context.architecturesList
-	} else {
-		architecturesList = packageList.Architectures()
-	}
-
-	if len(architecturesList) == 0 {
-		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
-	}
-
-	// Initial dependencies out of arguments
-	initialDependencies := make([]debian.Dependency, len(args)-3)
-	for i, arg := range args[3:] {
-		initialDependencies[i], err = debian.ParseDependency(arg)
-		if err != nil {
-			return fmt.Errorf("unable to parse argument: %s", err)
-		}
-	}
-
-	// Perform pull
-	for _, arch := range architecturesList {
-		dependencies := make([]debian.Dependency, len(initialDependencies), 128)
-		for i := range dependencies {
-			dependencies[i] = initialDependencies[i]
-			dependencies[i].Architecture = arch
-		}
-
-		// Go over list of initial dependencies + list of dependencies found
-		for i := 0; i < len(dependencies); i++ {
-			dep := dependencies[i]
-
-			// Search for package that can satisfy dependencies
-			pkg := sourcePackageList.Search(dep)
-			if pkg == nil {
-				color.Printf("@y[!]@| @!Dependency %s can't be satisfied with source %s@|", &dep, source)
-				fmt.Printf("\n")
-				continue
-			}
-
-			// Remove all packages with the same name and architecture
-			for p := packageList.Search(debian.Dependency{Architecture: arch, Pkg: pkg.Name}); p != nil; {
-				packageList.Remove(p)
-				color.Printf("@r[-]@| %s removed", p)
-				fmt.Printf("\n")
-				p = packageList.Search(debian.Dependency{Architecture: arch, Pkg: pkg.Name})
-			}
-
-			// Add new discovered package
-			packageList.Add(pkg)
-			color.Printf("@g[+]@| %s added", pkg)
-			fmt.Printf("\n")
-
-			if noDeps {
-				continue
-			}
-
-			// Find missing dependencies for single added package
-			pL := debian.NewPackageList()
-			pL.Add(pkg)
-
-			missing, err := pL.VerifyDependencies(context.dependencyOptions, []string{arch}, packageList)
-			if err != nil {
-				color.Printf("@y[!]@| @!Error while verifying dependencies for pkg %s: %s@|", pkg, err)
-				fmt.Printf("\n")
-			}
-
-			// Append missing dependencies to the list of dependencies to satisfy
-			for _, misDep := range missing {
-				found := false
-				for _, d := range dependencies {
-					if d == misDep {
-						found = true
-						break
-					}
-				}
-
-				if !found {
-					dependencies = append(dependencies, misDep)
-				}
-			}
-		}
-	}
-
-	if cmd.Flag.Lookup("dry-run").Value.Get().(bool) {
-		fmt.Printf("\nNot creating snapshot, as dry run was requested.\n")
-	} else {
-		// Create <destination> snapshot
-		destination := debian.NewSnapshotFromPackageList(args[2], []*debian.Snapshot{snapshot, source}, packageList,
-			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
-
-		err = snapshotCollection.Add(destination)
-		if err != nil {
-			return fmt.Errorf("unable to create snapshot: %s", err)
-		}
-
-		fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
-	}
-	return err
-}
-
-func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) != 2 {
-		cmd.Usage()
-		return err
-	}
-
-	onlyMatching := cmd.Flag.Lookup("only-matching").Value.Get().(bool)
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-	packageCollection := debian.NewPackageCollection(context.database)
-
-	// Load <name-a> snapshot
-	snapshotA, err := snapshotCollection.ByName(args[0])
-	if err != nil {
-		return fmt.Errorf("unable to load snapshot A: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(snapshotA)
-	if err != nil {
-		return fmt.Errorf("unable to load snapshot A: %s", err)
-	}
-
-	// Load <name-b> snapshot
-	snapshotB, err := snapshotCollection.ByName(args[1])
-	if err != nil {
-		return fmt.Errorf("unable to load snapshot B: %s", err)
-	}
-
-	err = snapshotCollection.LoadComplete(snapshotB)
-	if err != nil {
-		return fmt.Errorf("unable to load snapshot B: %s", err)
-	}
-
-	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), packageCollection)
-	if err != nil {
-		return fmt.Errorf("unable to calculate diff: %s", err)
-	}
-
-	if len(diff) == 0 {
-		fmt.Printf("Snapshots are identical.\n")
-	} else {
-		fmt.Printf("  Arch   | Package                                  | Version in A                             | Version in B\n")
-		for _, pdiff := range diff {
-			if onlyMatching && (pdiff.Left == nil || pdiff.Right == nil) {
-				continue
-			}
-
-			var verA, verB, pkg, arch, code string
-
-			if pdiff.Left == nil {
-				verA = "-"
-				verB = pdiff.Right.Version
-				pkg = pdiff.Right.Name
-				arch = pdiff.Right.Architecture
-			} else {
-				pkg = pdiff.Left.Name
-				arch = pdiff.Left.Architecture
-				verA = pdiff.Left.Version
-				if pdiff.Right == nil {
-					verB = "-"
-				} else {
-					verB = pdiff.Right.Version
-				}
-			}
-
-			if pdiff.Left == nil {
-				code = "@g+@|"
-			} else {
-				if pdiff.Right == nil {
-					code = "@r-@|"
-				} else {
-					code = "@y!@|"
-				}
-			}
-
-			color.Printf(code+" %-6s | %-40s | %-40s | %-40s\n", arch, pkg, verA, verB)
-		}
-	}
-
-	return err
-}
-
-func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
-	var err error
-	if len(args) < 2 {
-		cmd.Usage()
-		return err
-	}
-
-	snapshotCollection := debian.NewSnapshotCollection(context.database)
-
-	sources := make([]*debian.Snapshot, len(args)-1)
-
-	for i := 0; i < len(args)-1; i++ {
-		sources[i], err = snapshotCollection.ByName(args[i+1])
-		if err != nil {
-			return fmt.Errorf("unable to load snapshot: %s", err)
-		}
-
-		err = snapshotCollection.LoadComplete(sources[i])
-		if err != nil {
-			return fmt.Errorf("unable to load snapshot: %s", err)
-		}
-	}
-
-	result := sources[0].RefList()
-
-	for i := 1; i < len(sources); i++ {
-		result = result.Merge(sources[i].RefList())
-	}
-
-	sourceDescription := make([]string, len(sources))
-	for i, s := range sources {
-		sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
-	}
-
-	// Create <destination> snapshot
-	destination := debian.NewSnapshotFromRefList(args[0], sources, result,
-		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
-
-	err = snapshotCollection.Add(destination)
-	if err != nil {
-		return fmt.Errorf("unable to create snapshot: %s", err)
-	}
-
-	fmt.Printf("\nSnapshot %s successfully created.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
-
-	return err
-}
-
-func makeCmdSnapshotCreate() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotCreate,
-		UsageLine: "create <name> from mirror <mirror-name>",
-		Short:     "creates snapshot out of any mirror",
-		Long: `
-Command create makes persistent immutable snapshot of remote repository mirror. Snapshot could be
-published or further modified using merge, pull and other aptly features.
-
-ex.
-  $ aptly snapshot create wheezy-main-today from mirror wheezy-main
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-create", flag.ExitOnError),
-	}
-
-	return cmd
-
-}
-
-func makeCmdSnapshotList() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotList,
-		UsageLine: "list",
-		Short:     "lists snapshots",
-		Long: `
-Command list shows full list of snapshots created.
-
-ex:
-  $ aptly snapshot list
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-list", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdSnapshotShow() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotShow,
-		UsageLine: "show <name>",
-		Short:     "shows details about snapshot",
-		Long: `
-Command show displays full information about snapshot.
-
-ex.
-	$ aptly snapshot show wheezy-main
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-show", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdSnapshotVerify() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotVerify,
-		UsageLine: "verify <name> [<source> ...]",
-		Short:     "verifies that dependencies are satisfied in snapshot",
-		Long: `
-Verify does depenency resolution in snapshot, possibly using additional snapshots as dependency sources.
-All unsatisfied dependencies are returned.
-
-ex.
-	$ aptly snapshot verify wheezy-main wheezy-contrib wheezy-non-free
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-verify", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdSnapshotPull() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotPull,
-		UsageLine: "pull <name> <source> <destination> <package-name> ...",
-		Short:     "performs partial upgrades (pulls new packages) from another snapshot",
-		Long: `
-Command pull pulls new packages along with its dependencies in <name> snapshot
-from <source> snapshot. Also can upgrade package version from one snapshot into
-another, once again along with dependencies. New snapshot <destination> is created as result of this
-process. Packages could be specified simply as 'package-name' or as dependency 'package-name (>= version)'.
-
-ex.
-	$ aptly snapshot pull wheezy-main wheezy-backports wheezy-new-xorg xorg-server-server
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-pull", flag.ExitOnError),
-	}
-
-	cmd.Flag.Bool("dry-run", false, "don't create destination snapshot, just show what would be pulled")
-	cmd.Flag.Bool("no-deps", false, "don't process dependencies, just pull listed packages")
-
-	return cmd
-}
-
-func makeCmdSnapshotDiff() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotDiff,
-		UsageLine: "diff <name-a> <name-b>",
-		Short:     "calculates difference in packages between two snapshots",
-		Long: `
-Command diff shows list of missing and new packages, difference in package versions between two snapshots.
-
-ex.
-	$ aptly snapshot diff -only-matching wheezy-main wheezy-backports
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-diff", flag.ExitOnError),
-	}
-
-	cmd.Flag.Bool("only-matching", false, "display diff only for matching packages (don't display missing packages)")
-
-	return cmd
-}
-
-func makeCmdSnapshotMerge() *commander.Command {
-	cmd := &commander.Command{
-		Run:       aptlySnapshotMerge,
-		UsageLine: "merge <destination> <source> [<source>...]",
-		Short:     "merges snapshots into one, replacing matching packages",
-		Long: `
-Merge merges several snapshots into one. Merge happens from left to right. Packages with the same
-name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
-If run with only one source snapshot, merge copies source into destination.
-
-ex.
-	$ aptly snapshot merge wheezy-w-backports wheezy-main wheezy-backports
-`,
-		Flag: *flag.NewFlagSet("aptly-snapshot-merge", flag.ExitOnError),
-	}
-
-	return cmd
-}
-
-func makeCmdSnapshot() *commander.Command {
-	return &commander.Command{
-		UsageLine: "snapshot",
-		Short:     "manage snapshots of repositories",
-		Subcommands: []*commander.Command{
-			makeCmdSnapshotCreate(),
-			makeCmdSnapshotList(),
-			makeCmdSnapshotShow(),
-			makeCmdSnapshotVerify(),
-			makeCmdSnapshotPull(),
-			makeCmdSnapshotDiff(),
-			makeCmdSnapshotMerge(),
-			//makeCmdSnapshotDestroy(),
-		},
-		Flag: *flag.NewFlagSet("aptly-snapshot", flag.ExitOnError),
-	}
-}

console/progress.go

@@ -0,0 +1,180 @@
+package console
+
+import (
+	"fmt"
+	"github.com/cheggaaa/pb"
+	"github.com/smira/aptly/aptly"
+	"github.com/wsxiaoys/terminal/color"
+	"strings"
+)
+
+const (
+	codePrint = iota
+	codeProgress
+	codeHideProgress
+	codeStop
+	codeFlush
+)
+
+type printTask struct {
+	code    int
+	message string
+	reply   chan bool
+}
+
+// Progress is a progress displaying subroutine, it allows to show download and other operations progress
+// mixed with progress bar
+type Progress struct {
+	stop     chan bool
+	stopped  chan bool
+	queue    chan printTask
+	bar      *pb.ProgressBar
+	barShown bool
+}
+
+// Check interface
+var (
+	_ aptly.Progress = (*Progress)(nil)
+)
+
+// NewProgress creates new progress instance
+func NewProgress() *Progress {
+	return &Progress{
+		stopped: make(chan bool),
+		queue:   make(chan printTask, 100),
+	}
+}
+
+// Start makes progress start its work
+func (p *Progress) Start() {
+	go p.worker()
+}
+
+// Shutdown shuts down progress display
+func (p *Progress) Shutdown() {
+	p.ShutdownBar()
+	p.queue <- printTask{code: codeStop}
+	<-p.stopped
+}
+
+// Flush waits for all queued messages to be displayed
+func (p *Progress) Flush() {
+	ch := make(chan bool)
+	p.queue <- printTask{code: codeFlush, reply: ch}
+	<-ch
+}
+
+// InitBar starts progressbar for count bytes or count items
+func (p *Progress) InitBar(count int64, isBytes bool) {
+	if p.bar != nil {
+		panic("bar already initialized")
+	}
+	if RunningOnTerminal() {
+		p.bar = pb.New(0)
+		p.bar.Total = count
+		p.bar.NotPrint = true
+		p.bar.Callback = func(out string) {
+			p.queue <- printTask{code: codeProgress, message: out}
+		}
+
+		if isBytes {
+			p.bar.SetUnits(pb.U_BYTES)
+			p.bar.ShowSpeed = true
+		}
+		p.bar.Start()
+	}
+}
+
+// ShutdownBar stops progress bar and hides it
+func (p *Progress) ShutdownBar() {
+	if p.bar == nil {
+		return
+	}
+	p.bar.Finish()
+	p.bar = nil
+	p.queue <- printTask{code: codeHideProgress}
+}
+
+// Write is implementation of io.Writer to support updating of progress bar
+func (p *Progress) Write(s []byte) (int, error) {
+	if p.bar != nil {
+		p.bar.Add(len(s))
+	}
+	return len(s), nil
+}
+
+// AddBar increments progress for progress bar
+func (p *Progress) AddBar(count int) {
+	if p.bar != nil {
+		p.bar.Add(count)
+	}
+}
+
+// SetBar sets current position for progress bar
+func (p *Progress) SetBar(count int) {
+	if p.bar != nil {
+		p.bar.Set(count)
+	}
+}
+
+// Printf does printf but in safe manner: not overwriting progress bar
+func (p *Progress) Printf(msg string, a ...interface{}) {
+	p.queue <- printTask{code: codePrint, message: fmt.Sprintf(msg, a...)}
+}
+
+// ColoredPrintf does printf in colored way + newline
+func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
+	if RunningOnTerminal() {
+		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
+	} else {
+		// stip color marks
+		var prev rune
+		msg = strings.Map(func(r rune) rune {
+			if prev == '@' {
+				prev = 0
+				if r == '@' {
+					return r
+				}
+				return -1
+			}
+			prev = r
+			if r == '@' {
+				return -1
+
+			}
+
+			return r
+		}, msg)
+
+		p.Printf(msg+"\n", a...)
+	}
+}
+
+func (p *Progress) worker() {
+	for {
+		task := <-p.queue
+		switch task.code {
+		case codePrint:
+			if p.barShown {
+				fmt.Print("\r\033[2K")
+				p.barShown = false
+			}
+			fmt.Print(task.message)
+		case codeProgress:
+			if p.bar != nil {
+				fmt.Print("\r" + task.message)
+				p.barShown = true
+			}
+		case codeHideProgress:
+			if p.barShown {
+				fmt.Print("\r\033[2K")
+				p.barShown = false
+			}
+		case codeFlush:
+			task.reply <- true
+		case codeStop:
+			p.stopped <- true
+			return
+		}
+	}
+}

console/terminal.go

@@ -0,0 +1,13 @@
+// +build !freebsd
+
+package console
+
+import (
+	"code.google.com/p/go.crypto/ssh/terminal"
+	"syscall"
+)
+
+// RunningOnTerminal checks whether stdout is terminal
+func RunningOnTerminal() bool {
+	return terminal.IsTerminal(syscall.Stdout)
+}

console/terminal_bsd.go

@@ -0,0 +1,10 @@
+// +build freebsd
+
+package console
+
+// RunningOnTerminal checks whether stdout is terminal
+//
+// Stub for FreeBSD, until in go1.3 terminal.IsTerminal would start working for FreeBSD
+func RunningOnTerminal() bool {
+	return false
+}

database/leveldb.go

@@ -19,12 +19,16 @@ type Storage interface {
 	Get(key []byte) ([]byte, error)
 	Put(key []byte, value []byte) error
 	Delete(key []byte) error
+	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
+	StartBatch()
+	FinishBatch() error
 }
 
 type levelDB struct {
-	db *leveldb.DB
+	db    *leveldb.DB
+	batch *leveldb.Batch
 }
 
 // Check interface
@@ -58,27 +62,58 @@ func (l *levelDB) Get(key []byte) ([]byte, error) {
 }
 
 func (l *levelDB) Put(key []byte, value []byte) error {
+	if l.batch != nil {
+		l.batch.Put(key, value)
+		return nil
+	}
+	old, err := l.db.Get(key, nil)
+	if err != nil {
+		if err != leveldb.ErrNotFound {
+			return err
+		}
+	} else {
+		if bytes.Compare(old, value) == 0 {
+			return nil
+		}
+	}
 	return l.db.Put(key, value, nil)
 }
 
 func (l *levelDB) Delete(key []byte) error {
+	if l.batch != nil {
+		l.batch.Delete(key)
+		return nil
+	}
 	return l.db.Delete(key, nil)
 }
 
+func (l *levelDB) KeysByPrefix(prefix []byte) [][]byte {
+	result := make([][]byte, 0, 20)
+
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		key := iterator.Key()
+		keyc := make([]byte, len(key))
+		copy(keyc, key)
+		result = append(result, keyc)
+	}
+
+	return result
+}
+
 func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 	result := make([][]byte, 0, 20)
 
-	iterator := l.db.NewIterator(nil)
-	if iterator.Seek(prefix) {
-		for bytes.HasPrefix(iterator.Key(), prefix) {
-			val := iterator.Value()
-			valc := make([]byte, len(val))
-			copy(valc, val)
-			result = append(result, valc)
-			if !iterator.Next() {
-				break
-			}
-		}
+	iterator := l.db.NewIterator(nil, nil)
+	defer iterator.Release()
+
+	for ok := iterator.Seek(prefix); ok && bytes.HasPrefix(iterator.Key(), prefix); ok = iterator.Next() {
+		val := iterator.Value()
+		valc := make([]byte, len(val))
+		copy(valc, val)
+		result = append(result, valc)
 	}
 
 	return result
@@ -87,3 +122,19 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 func (l *levelDB) Close() error {
 	return l.db.Close()
 }
+
+func (l *levelDB) StartBatch() {
+	if l.batch != nil {
+		panic("batch already started")
+	}
+	l.batch = new(leveldb.Batch)
+}
+
+func (l *levelDB) FinishBatch() error {
+	if l.batch == nil {
+		panic("no batch")
+	}
+	err := l.db.Write(l.batch, nil)
+	l.batch = nil
+	return err
+}

database/leveldb_test.go

@@ -59,21 +59,66 @@ func (s *LevelDBSuite) TestDelete(c *C) {
 
 	_, err = s.db.Get(key)
 	c.Assert(err, ErrorMatches, "key not found")
+
+	err = s.db.Delete(key)
+	c.Assert(err, IsNil)
 }
 
-func (s *LevelDBSuite) TestFetchByPrefix(c *C) {
+func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
 
 	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
 	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
 	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
 	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
 	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
+	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
 	c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+	c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
+}
+
+func (s *LevelDBSuite) TestBatch(c *C) {
+	var (
+		key    = []byte("key")
+		key2   = []byte("key2")
+		value  = []byte("value")
+		value2 = []byte("value2")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	s.db.StartBatch()
+	s.db.Put(key2, value2)
+	s.db.Delete(key)
+
+	v, err := s.db.Get(key)
+	c.Check(err, IsNil)
+	c.Check(v, DeepEquals, value)
+
+	_, err = s.db.Get(key2)
+	c.Check(err, ErrorMatches, "key not found")
+
+	err = s.db.FinishBatch()
+	c.Check(err, IsNil)
+
+	v2, err := s.db.Get(key2)
+	c.Check(err, IsNil)
+	c.Check(v2, DeepEquals, value2)
+
+	_, err = s.db.Get(key)
+	c.Check(err, ErrorMatches, "key not found")
+
+	c.Check(func() { s.db.FinishBatch() }, Panics, "no batch")
+
+	s.db.StartBatch()
+	c.Check(func() { s.db.StartBatch() }, Panics, "batch already started")
 }

debian/deb.go

@@ -0,0 +1,99 @@
+package debian
+
+import (
+	"archive/tar"
+	"bufio"
+	"compress/gzip"
+	"fmt"
+	"github.com/mkrautz/goar"
+	"github.com/smira/aptly/utils"
+	"io"
+	"os"
+	"strings"
+)
+
+// GetControlFileFromDeb reads control file from deb package
+func GetControlFileFromDeb(packageFile string) (Stanza, error) {
+	file, err := os.Open(packageFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	library := ar.NewReader(file)
+	for {
+		header, err := library.Next()
+		if err == io.EOF {
+			return nil, fmt.Errorf("unable to find control.tar.gz part")
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read .deb archive: %s", err)
+		}
+
+		if header.Name == "control.tar.gz" {
+			ungzip, err := gzip.NewReader(library)
+			if err != nil {
+				return nil, fmt.Errorf("unable to ungzip: %s", err)
+			}
+			defer ungzip.Close()
+
+			untar := tar.NewReader(ungzip)
+			for {
+				tarHeader, err := untar.Next()
+				if err == io.EOF {
+					return nil, fmt.Errorf("unable to find control file")
+				}
+				if err != nil {
+					return nil, fmt.Errorf("unable to read .tar archive: %s", err)
+				}
+
+				if tarHeader.Name == "./control" {
+					reader := NewControlFileReader(untar)
+					stanza, err := reader.ReadStanza()
+					if err != nil {
+						return nil, err
+					}
+
+					return stanza, nil
+				}
+			}
+		}
+	}
+}
+
+// GetControlFileFromDsc reads control file from dsc package
+func GetControlFileFromDsc(dscFile string, verifier utils.Verifier) (Stanza, error) {
+	file, err := os.Open(dscFile)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	line, err := bufio.NewReader(file).ReadString('\n')
+	if err != nil {
+		return nil, err
+	}
+
+	file.Seek(0, 0)
+
+	var text *os.File
+
+	if strings.Index(line, "BEGIN PGP SIGN") != -1 {
+		text, err = verifier.ExtractClearsigned(file)
+		if err != nil {
+			return nil, err
+		}
+		defer text.Close()
+	} else {
+		text = file
+	}
+
+	reader := NewControlFileReader(text)
+	stanza, err := reader.ReadStanza()
+	if err != nil {
+		return nil, err
+	}
+
+	return stanza, nil
+
+}

debian/deb_test.go

@@ -0,0 +1,56 @@
+package debian
+
+import (
+	"github.com/smira/aptly/utils"
+	. "launchpad.net/gocheck"
+	"path/filepath"
+	"runtime"
+)
+
+type DebSuite struct {
+	debFile, dscFile, dscFileNoSign string
+}
+
+var _ = Suite(&DebSuite{})
+
+func (s *DebSuite) SetUpSuite(c *C) {
+	_, _File, _, _ := runtime.Caller(0)
+	s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+	s.dscFile = filepath.Join(filepath.Dir(_File), "../system/files/pyspi_0.6.1-1.3.dsc")
+	s.dscFileNoSign = filepath.Join(filepath.Dir(_File), "../system/files/pyspi-0.6.1-1.3.stripped.dsc")
+}
+
+func (s *DebSuite) TestGetControlFileFromDeb(c *C) {
+	_, err := GetControlFileFromDeb("/no/such/file")
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+
+	_, _File, _, _ := runtime.Caller(0)
+	_, err = GetControlFileFromDeb(_File)
+	c.Check(err, ErrorMatches, "unable to read .deb archive: ar: missing global header")
+
+	st, err := GetControlFileFromDeb(s.debFile)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "1.49.0.1")
+	c.Check(st["Package"], Equals, "libboost-program-options-dev")
+}
+
+func (s *DebSuite) TestGetControlFileFromDsc(c *C) {
+	verifier := &utils.GpgVerifier{}
+
+	_, err := GetControlFileFromDsc("/no/such/file", verifier)
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+
+	_, _File, _, _ := runtime.Caller(0)
+	_, err = GetControlFileFromDsc(_File, verifier)
+	c.Check(err, ErrorMatches, "malformed stanza syntax")
+
+	st, err := GetControlFileFromDsc(s.dscFile, verifier)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "0.6.1-1.3")
+	c.Check(st["Source"], Equals, "pyspi")
+
+	st, err = GetControlFileFromDsc(s.dscFileNoSign, verifier)
+	c.Check(err, IsNil)
+	c.Check(st["Version"], Equals, "0.6.1-1.4")
+	c.Check(st["Source"], Equals, "pyspi")
+}

debian/debian.go

@@ -0,0 +1,2 @@
+// Package debian implements Debian-specific repository handling
+package debian

debian/debian_test.go

@@ -0,0 +1,11 @@
+package debian
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

debian/format.go

@@ -11,7 +11,8 @@ import (
 type Stanza map[string]string
 
 // Canonical order of fields in stanza
-var canocialOrder = []string{"Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer", "Architecture"}
+var canocialOrder = []string{"Origin", "Label", "Suite", "Package", "Version", "Installed-Size", "Priority", "Section", "Maintainer",
+	"Architecture", "Codename", "Date", "Architectures", "Components", "Description", "MD5sum", "MD5Sum", "SHA1", "SHA256"}
 
 // Copy returns copy of Stanza
 func (s Stanza) Copy() (result Stanza) {

debian/list.go

@@ -1,11 +1,11 @@
 package debian
 
 import (
-	"bytes"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
-	"github.com/ugorji/go/codec"
 	"sort"
+	"strings"
 )
 
 // Dependency options
@@ -18,6 +18,8 @@ const (
 	DepFollowRecommends
 	// DepFollowAllVariants follows all variants if depends on "a | b"
 	DepFollowAllVariants
+	// DepFollowBuild pulls build dependencies
+	DepFollowBuild
 )
 
 // PackageList is list of unique (by key) packages
@@ -48,17 +50,33 @@ func NewPackageList() *PackageList {
 }
 
 // NewPackageListFromRefList loads packages list from PackageRefList
-func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageCollection) (*PackageList, error) {
+func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageCollection, progress aptly.Progress) (*PackageList, error) {
+	// empty reflist
+	if reflist == nil {
+		return NewPackageList(), nil
+	}
+
 	result := &PackageList{packages: make(map[string]*Package, reflist.Len())}
 
+	if progress != nil {
+		progress.InitBar(int64(reflist.Len()), false)
+	}
+
 	err := reflist.ForEach(func(key []byte) error {
 		p, err := collection.ByKey(key)
 		if err != nil {
 			return fmt.Errorf("unable to load package with key %s: %s", key, err)
 		}
+		if progress != nil {
+			progress.AddBar(1)
+		}
 		return result.Add(p)
 	})
 
+	if progress != nil {
+		progress.ShutdownBar()
+	}
+
 	if err != nil {
 		return nil, err
 	}
@@ -68,7 +86,7 @@ func NewPackageListFromRefList(reflist *PackageRefList, collection *PackageColle
 
 // Add appends package to package list, additionally checking for uniqueness
 func (l *PackageList) Add(p *Package) error {
-	key := string(p.Key())
+	key := string(p.Key(""))
 	existing, ok := l.packages[key]
 	if ok {
 		if !existing.Equals(p) {
@@ -131,7 +149,7 @@ func (l *PackageList) Append(pl *PackageList) error {
 
 // Remove removes package from the list, and updates index when required
 func (l *PackageList) Remove(p *Package) {
-	delete(l.packages, string(p.Key()))
+	delete(l.packages, string(p.Key("")))
 	if l.indexed {
 		for _, provides := range p.Provides {
 			for i, pkg := range l.providesIndex[provides] {
@@ -158,11 +176,13 @@ func (l *PackageList) Remove(p *Package) {
 	}
 }
 
-// Architectures returns list of architectures present in packages
-func (l *PackageList) Architectures() (result []string) {
+// Architectures returns list of architectures present in packages and flag if source packages are present.
+//
+// If includeSource is true, meta-architecture "source" would be present in the list
+func (l *PackageList) Architectures(includeSource bool) (result []string) {
 	result = make([]string, 0, 10)
 	for _, pkg := range l.packages {
-		if pkg.Architecture != "all" && !utils.StrSliceHasItem(result, pkg.Architecture) {
+		if pkg.Architecture != "all" && (pkg.Architecture != "source" || includeSource) && !utils.StrSliceHasItem(result, pkg.Architecture) {
 			result = append(result, pkg.Architecture)
 		}
 	}
@@ -199,13 +219,21 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 // VerifyDependencies looks for missing dependencies in package list.
 //
 // Analysis would be peformed for each architecture, in specified sources
-func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList) ([]Dependency, error) {
+func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
 	missing := make([]Dependency, 0, 128)
 
+	if progress != nil {
+		progress.InitBar(int64(l.Len())*int64(len(architectures)), false)
+	}
+
 	for _, arch := range architectures {
 		cache := make(map[string]bool, 2048)
 
 		for _, p := range l.packages {
+			if progress != nil {
+				progress.AddBar(1)
+			}
+
 			if !p.MatchesArchitecture(arch) {
 				continue
 			}
@@ -222,7 +250,9 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 				missingCount := 0
 
 				for _, dep := range variants {
-					dep.Architecture = arch
+					if dep.Architecture == "" {
+						dep.Architecture = arch
+					}
 
 					hash := dep.Hash()
 					r, ok := cache[hash]
@@ -258,6 +288,10 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 		}
 	}
 
+	if progress != nil {
+		progress.ShutdownBar()
+	}
+
 	return missing, nil
 }
 
@@ -309,269 +343,95 @@ func (l *PackageList) Search(dep Dependency) *Package {
 
 	for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
 		p := l.packagesIndex[i]
-		if p.MatchesArchitecture(dep.Architecture) {
-			if dep.Relation == VersionDontCare {
-				return p
-			}
-
-			r := CompareVersions(p.Version, dep.Version)
-			switch dep.Relation {
-			case VersionEqual:
-				if r == 0 {
-					return p
-				}
-			case VersionLess:
-				if r < 0 {
-					return p
-				}
-			case VersionGreater:
-				if r > 0 {
-					return p
-				}
-			case VersionLessOrEqual:
-				if r <= 0 {
-					return p
-				}
-			case VersionGreaterOrEqual:
-				if r >= 0 {
-					return p
-				}
-			}
+		if p.MatchesDependency(dep) {
+			return p
 		}
+
 		i++
 	}
 	return nil
 }
 
-// PackageRefList is a list of keys of packages, this is basis for snapshot
-// and similar stuff
-//
-// Refs are sorted in lexicographical order
-type PackageRefList struct {
-	// List of package keys
-	Refs [][]byte
-}
-
-// Verify interface
-var (
-	_ sort.Interface = &PackageRefList{}
-)
-
-// NewPackageRefListFromPackageList creates PackageRefList from PackageList
-func NewPackageRefListFromPackageList(list *PackageList) *PackageRefList {
-	reflist := &PackageRefList{}
-	reflist.Refs = make([][]byte, list.Len())
-
-	i := 0
-	for _, p := range list.packages {
-		reflist.Refs[i] = p.Key()
-		i++
+// Filter filters package index by specified queries (ORed together), possibly pulling dependencies
+func (l *PackageList) Filter(queries []string, withDependencies bool, source *PackageList, dependencyOptions int, architecturesList []string) (*PackageList, error) {
+	if !l.indexed {
+		panic("list not indexed, can't filter")
 	}
 
-	sort.Sort(reflist)
+	result := NewPackageList()
 
-	return reflist
-}
+	for _, query := range queries {
+		isDepQuery := strings.IndexAny(query, " (){}=<>") != -1
 
-// Len returns number of refs
-func (l *PackageRefList) Len() int {
-	return len(l.Refs)
-}
+		if !isDepQuery {
+			// try to interpret query as package string representation
 
-// Swap swaps two refs
-func (l *PackageRefList) Swap(i, j int) {
-	l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i]
-}
+			// convert Package.String() to Package.Key()
+			i := strings.Index(query, "_")
+			if i != -1 {
+				pkg, query := query[:i], query[i+1:]
+				j := strings.LastIndex(query, "_")
+				if j != -1 {
+					version, arch := query[:j], query[j+1:]
+					p := l.packages["P"+arch+" "+pkg+" "+version]
+					if p != nil {
+						result.Add(p)
+						continue
+					}
+				}
+			}
+		}
 
-// Compare compares two refs in lexographical order
-func (l *PackageRefList) Less(i, j int) bool {
-	return bytes.Compare(l.Refs[i], l.Refs[j]) < 0
-}
-
-// Encode does msgpack encoding of PackageRefList
-func (l *PackageRefList) Encode() []byte {
-	var buf bytes.Buffer
-
-	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(l)
-
-	return buf.Bytes()
-}
-
-// Decode decodes msgpack representation into PackageRefLit
-func (l *PackageRefList) Decode(input []byte) error {
-	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(l)
-}
-
-// ForEach calls handler for each package ref in list
-func (l *PackageRefList) ForEach(handler func([]byte) error) error {
-	var err error
-	for _, p := range l.Refs {
-		err = handler(p)
+		// try as dependency
+		dep, err := ParseDependency(query)
 		if err != nil {
-			return err
-		}
-	}
-	return err
-}
-
-// PackageDiff is a difference between two packages in a list.
-//
-// If left & right are present, difference is in package version
-// If left is nil, package is present only in right
-// If right is nil, package is present only in left
-type PackageDiff struct {
-	Left, Right *Package
-}
-
-// PackageDiffs is a list of PackageDiff records
-type PackageDiffs []PackageDiff
-
-// Diff calculates difference between two reflists
-func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection) (result PackageDiffs, err error) {
-	result = make(PackageDiffs, 0, 128)
-
-	// pointer to left and right reflists
-	il, ir := 0, 0
-	// length of reflists
-	ll, lr := l.Len(), r.Len()
-	// cached loaded packages on the left & right
-	pl, pr := (*Package)(nil), (*Package)(nil)
-
-	// until we reached end of both lists
-	for il < ll || ir < lr {
-		// if we've exhausted left list, pull the rest from the right
-		if il == ll {
-			pr, err = packageCollection.ByKey(r.Refs[ir])
-			if err != nil {
+			if isDepQuery {
 				return nil, err
 			}
-			result = append(result, PackageDiff{Left: nil, Right: pr})
-			ir++
-			continue
-		}
-		// if we've exhausted right list, pull the rest from the left
-		if ir == lr {
-			pl, err = packageCollection.ByKey(l.Refs[il])
-			if err != nil {
-				return nil, err
-			}
-			result = append(result, PackageDiff{Left: pl, Right: nil})
-			il++
+			// parsing failed, but probably that wasn't a dep query
 			continue
 		}
 
-		// refs on both sides are present, load them
-		rl, rr := l.Refs[il], r.Refs[ir]
-		// compare refs
-		rel := bytes.Compare(rl, rr)
+		i := sort.Search(len(l.packagesIndex), func(j int) bool { return l.packagesIndex[j].Name >= dep.Pkg })
 
-		if rel == 0 {
-			// refs are identical, so are packages, advance pointer
-			il++
-			ir++
-			pl, pr = nil, nil
-		} else {
-			// load pl & pr if they haven't been loaded before
-			if pl == nil {
-				pl, err = packageCollection.ByKey(rl)
-				if err != nil {
-					return nil, err
-				}
+		for i < len(l.packagesIndex) && l.packagesIndex[i].Name == dep.Pkg {
+			p := l.packagesIndex[i]
+			if p.MatchesDependency(dep) {
+				result.Add(p)
 			}
-
-			if pr == nil {
-				pr, err = packageCollection.ByKey(rr)
-				if err != nil {
-					return nil, err
-				}
-			}
-
-			// is pl & pr the same package, but different version?
-			if pl.Name == pr.Name && pl.Architecture == pr.Architecture {
-				result = append(result, PackageDiff{Left: pl, Right: pr})
-				il++
-				ir++
-				pl, pr = nil, nil
-			} else {
-				// otherwise pl or pr is missing on one of the sides
-				if rel < 0 {
-					result = append(result, PackageDiff{Left: pl, Right: nil})
-					il++
-					pl = nil
-				} else {
-					result = append(result, PackageDiff{Left: nil, Right: pr})
-					ir++
-					pr = nil
-				}
-			}
-
+			i++
 		}
 	}
 
-	return
-}
+	if withDependencies {
+		added := result.Len()
 
-// Merge merges reflist r into current reflist. Merge replaces matching packages (by architecture/name)
-// with reference from r.
-func (l *PackageRefList) Merge(r *PackageRefList) (result *PackageRefList) {
-	// pointer to left and right reflists
-	il, ir := 0, 0
-	// length of reflists
-	ll, lr := l.Len(), r.Len()
+		dependencySource := NewPackageList()
+		dependencySource.Append(source)
+		dependencySource.Append(result)
+		dependencySource.PrepareIndex()
 
-	result = &PackageRefList{}
-	result.Refs = make([][]byte, 0, ll+lr)
+		// while some new dependencies were discovered
+		for added > 0 {
+			added = 0
 
-	// until we reached end of both lists
-	for il < ll || ir < lr {
-		// if we've exhausted left list, pull the rest from the right
-		if il == ll {
-			result.Refs = append(result.Refs, r.Refs[ir:]...)
-			break
-		}
-		// if we've exhausted right list, pull the rest from the left
-		if ir == lr {
-			result.Refs = append(result.Refs, l.Refs[il:]...)
-			break
-		}
-
-		// refs on both sides are present, load them
-		rl, rr := l.Refs[il], r.Refs[ir]
-		// compare refs
-		rel := bytes.Compare(rl, rr)
-
-		if rel == 0 {
-			// refs are identical, so are packages, advance pointer
-			result.Refs = append(result.Refs, l.Refs[il])
-			il++
-			ir++
-		} else {
-			partsL := bytes.Split(rl, []byte(" "))
-			archL, nameL := partsL[0][1:], partsL[1]
-
-			partsR := bytes.Split(rr, []byte(" "))
-			archR, nameR := partsR[0][1:], partsR[1]
-
-			if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 {
-				// override with package from the right
-				result.Refs = append(result.Refs, r.Refs[ir])
-				il++
-				ir++
-			} else {
-				// otherwise append smallest of two
-				if rel < 0 {
-					result.Refs = append(result.Refs, l.Refs[il])
-					il++
-				} else {
-					result.Refs = append(result.Refs, r.Refs[ir])
-					ir++
-				}
+			// find missing dependencies
+			missing, err := result.VerifyDependencies(dependencyOptions, architecturesList, dependencySource, nil)
+			if err != nil {
+				return nil, err
 			}
 
+			// try to satisfy dependencies
+			for _, dep := range missing {
+				p := l.Search(dep)
+				if p != nil {
+					result.Add(p)
+					dependencySource.Add(p)
+					added++
+				}
+			}
 		}
 	}
 
-	return
+	return result, nil
 }

debian/list_test.go

@@ -2,9 +2,9 @@ package debian
 
 import (
 	"errors"
-	"github.com/smira/aptly/database"
 	. "launchpad.net/gocheck"
 	"sort"
+	"strings"
 )
 
 type PackageListSuite struct {
@@ -13,8 +13,9 @@ type PackageListSuite struct {
 	p1, p2, p3, p4, p5, p6 *Package
 
 	// Mocked packages in list
-	packages []*Package
-	il       *PackageList
+	packages       []*Package
+	sourcePackages []*Package
+	il             *PackageList
 }
 
 var _ = Suite(&PackageListSuite{})
@@ -39,24 +40,33 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 
 	s.il = NewPackageList()
 	s.packages = []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}},
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}},
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all", PreDepends: []string{"dpkg (>= 1.6)"}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}},
-		&Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Provides: []string{"mail-agent"}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}},
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}},
-		&Package{Name: "app", Version: "1.0", Architecture: "s390", PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}},
-		&Package{Name: "aa", Version: "2.0-1", Architecture: "i386", PreDepends: []string{"dpkg (>= 1.6)"}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}},
-		&Package{Name: "libx", Version: "1.5", Architecture: "arm", PreDepends: []string{"dpkg (>= 1.6)"}},
-		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}},
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386", Source: "lib (0.9)", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"mail-agent"}}},
+		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all", Source: "app", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "mailer", Version: "3.5.8", Architecture: "i386", Source: "postfix (1.3)", Provides: []string{"mail-agent"}, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}, Depends: []string{"lib (>> 0.9) | libx (>= 1.5)", "data (>= 1.0) | mail-agent"}}},
+		&Package{Name: "app", Version: "1.0", Architecture: "s390", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
+		&Package{Name: "aa", Version: "2.0-1", Architecture: "i386", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "amd64", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		&Package{Name: "libx", Version: "1.5", Architecture: "arm", deps: &PackageDependencies{PreDepends: []string{"dpkg (>= 1.6)"}}},
+		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "arm", Provides: []string{"package-installer"}, deps: &PackageDependencies{}},
+		&Package{Name: "dpkg", Version: "1.6.1-3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		&Package{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 	for _, p := range s.packages {
 		s.il.Add(p)
 	}
 	s.il.PrepareIndex()
 
+	s.sourcePackages = []*Package{
+		&Package{Name: "postfix", Version: "1.3", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		&Package{Name: "aa", Version: "2.0-1", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+		&Package{Name: "lib", Version: "0.9", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
+	}
+
 }
 
 func (s *PackageListSuite) TestAddLen(c *C) {
@@ -110,14 +120,14 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
 	for i, p := range s.il.packagesIndex {
 		names[i] = p.Name
 	}
-	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "libx", "mailer"})
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "dpkg", "libx", "mailer"})
 
 	s.il.Remove(s.packages[4])
 	names = make([]string, s.il.Len())
 	for i, p := range s.il.packagesIndex {
 		names[i] = p.Name
 	}
-	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "libx"})
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "dpkg", "libx"})
 	c.Check(s.il.providesIndex["mail-agent"], DeepEquals, []*Package{})
 
 	s.il.Remove(s.packages[9])
@@ -125,7 +135,7 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
 	for i, p := range s.il.packagesIndex {
 		names[i] = p.Name
 	}
-	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "libx"})
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "dpkg", "libx"})
 	c.Check(s.il.providesIndex["package-installer"], HasLen, 2)
 
 	s.il.Remove(s.packages[1])
@@ -133,7 +143,7 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
 	for i, p := range s.il.packagesIndex {
 		names[i] = p.Name
 	}
-	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "libx"})
+	c.Check(names, DeepEquals, []string{"aa", "app", "app", "app", "app", "data", "dpkg", "dpkg", "dpkg", "libx"})
 	c.Check(s.il.providesIndex["package-installer"], DeepEquals, []*Package{s.packages[11]})
 }
 
@@ -173,7 +183,7 @@ func (s *PackageListSuite) TestAppend(c *C) {
 
 	err := s.list.Append(s.il)
 	c.Check(err, IsNil)
-	c.Check(s.list.Len(), Equals, 14)
+	c.Check(s.list.Len(), Equals, 16)
 
 	list := NewPackageList()
 	list.Add(s.p4)
@@ -210,240 +220,84 @@ func (s *PackageListSuite) TestSearch(c *C) {
 	c.Check(s.il.Search(Dependency{Architecture: "i386", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}), IsNil)
 }
 
-func (s *PackageListSuite) TestVerifyDependencies(c *C) {
-	missing, err := s.il.VerifyDependencies(0, []string{"i386"}, s.il)
+func (s *PackageListSuite) TestFilter(c *C) {
+	c.Check(func() { s.list.Filter([]string{"abcd_0.3_i386"}, false, nil, 0, nil) }, Panics, "list not indexed, can't filter")
 
+	_, err := s.il.Filter([]string{"app >3)"}, false, nil, 0, nil)
+	c.Check(err, ErrorMatches, "unable to parse dependency.*")
+
+	plString := func(l *PackageList) string {
+		list := make([]string, 0, l.Len())
+		for _, p := range l.packages {
+			list = append(list, p.String())
+		}
+
+		sort.Strings(list)
+
+		return strings.Join(list, " ")
+	}
+
+	result, err := s.il.Filter([]string{"app_1.1~bp1_i386"}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386")
+
+	result, err = s.il.Filter([]string{"app_1.1~bp1_i386", "dpkg_1.7_source", "dpkg_1.8_amd64"}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]string{"app", "dpkg (>>1.6.1-3)", "app (>=1.0)", "xyz", "aa (>>3.0)"}, false, nil, 0, nil)
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 dpkg_1.7_i386 dpkg_1.7_source")
+
+	result, err = s.il.Filter([]string{"app {i386}"}, true, NewPackageList(), 0, []string{"i386"})
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+
+	result, err = s.il.Filter([]string{"app (>=0.9)", "lib", "data"}, true, NewPackageList(), 0, []string{"i386", "amd64"})
+	c.Check(err, IsNil)
+	c.Check(plString(result), Equals, "app_1.0_s390 app_1.1~bp1_amd64 app_1.1~bp1_arm app_1.1~bp1_i386 data_1.1~bp1_all dpkg_1.6.1-3_amd64 dpkg_1.7_i386 lib_1.0_i386 mailer_3.5.8_i386")
+}
+
+func (s *PackageListSuite) TestVerifyDependencies(c *C) {
+	missing, err := s.il.VerifyDependencies(0, []string{"i386"}, s.il, nil)
 	c.Check(err, IsNil)
 	c.Check(missing, DeepEquals, []Dependency{})
 
-	missing, err = s.il.VerifyDependencies(0, []string{"i386", "amd64"}, s.il)
-
+	missing, err = s.il.VerifyDependencies(0, []string{"i386", "amd64"}, s.il, nil)
 	c.Check(err, IsNil)
 	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
 
-	missing, err = s.il.VerifyDependencies(0, []string{"arm"}, s.il)
-
+	missing, err = s.il.VerifyDependencies(0, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
 	c.Check(missing, DeepEquals, []Dependency{})
 
-	missing, err = s.il.VerifyDependencies(DepFollowAllVariants, []string{"arm"}, s.il)
-
+	missing, err = s.il.VerifyDependencies(DepFollowAllVariants, []string{"arm"}, s.il, nil)
 	c.Check(err, IsNil)
 	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "arm"},
 		Dependency{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
 
-	_, err = s.il.VerifyDependencies(0, []string{"i386", "amd64", "s390"}, s.il)
+	for _, p := range s.sourcePackages {
+		s.il.Add(p)
+	}
 
-	c.Check(err, ErrorMatches, "unable to process package app-1.0_s390:.*")
+	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "lib", Relation: VersionGreater, Version: "0.9", Architecture: "amd64"}})
+
+	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"arm"}, s.il, nil)
+	c.Check(err, IsNil)
+	c.Check(missing, DeepEquals, []Dependency{Dependency{Pkg: "libx", Relation: VersionEqual, Version: "1.5", Architecture: "source"}})
+
+	_, err = s.il.VerifyDependencies(0, []string{"i386", "amd64", "s390"}, s.il, nil)
+	c.Check(err, ErrorMatches, "unable to process package app_1.0_s390:.*")
 }
 
 func (s *PackageListSuite) TestArchitectures(c *C) {
-	archs := s.il.Architectures()
+	archs := s.il.Architectures(true)
+	sort.Strings(archs)
+	c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390", "source"})
+
+	archs = s.il.Architectures(false)
 	sort.Strings(archs)
 	c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
 }
-
-func (s *PackageListSuite) TestNewPackageListFromRefList(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
-	coll := NewPackageCollection(db)
-	coll.Update(s.p1)
-	coll.Update(s.p3)
-
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
-
-	reflist := NewPackageRefListFromPackageList(s.list)
-
-	_, err := NewPackageListFromRefList(reflist, coll)
-	c.Assert(err, ErrorMatches, "unable to load package with key.*")
-
-	coll.Update(s.p5)
-	coll.Update(s.p6)
-
-	list, err := NewPackageListFromRefList(reflist, coll)
-	c.Assert(err, IsNil)
-	c.Check(list.Len(), Equals, 4)
-	c.Check(list.Add(s.p4), ErrorMatches, "conflict in package.*")
-}
-
-func (s *PackageListSuite) TestNewPackageRefList(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
-
-	reflist := NewPackageRefListFromPackageList(s.list)
-	c.Assert(reflist.Len(), Equals, 4)
-	c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key()))
-	c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key()))
-	c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key()))
-	c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key()))
-}
-
-func (s *PackageListSuite) TestPackageRefListEncodeDecode(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
-
-	reflist := NewPackageRefListFromPackageList(s.list)
-
-	reflist2 := &PackageRefList{}
-	err := reflist2.Decode(reflist.Encode())
-	c.Assert(err, IsNil)
-	c.Check(reflist2.Len(), Equals, reflist.Len())
-	c.Check(reflist2.Refs, DeepEquals, reflist.Refs)
-}
-
-func (s *PackageListSuite) TestPackageRefListForeach(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
-
-	reflist := NewPackageRefListFromPackageList(s.list)
-
-	Len := 0
-	err := reflist.ForEach(func([]byte) error {
-		Len++
-		return nil
-	})
-
-	c.Check(Len, Equals, 4)
-	c.Check(err, IsNil)
-
-	e := errors.New("b")
-
-	err = reflist.ForEach(func([]byte) error {
-		return e
-	})
-
-	c.Check(err, Equals, e)
-}
-
-func (s *PackageListSuite) TestDiff(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
-	coll := NewPackageCollection(db)
-
-	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
-		&Package{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
-	}
-
-	for _, p := range packages {
-		coll.Update(p)
-	}
-
-	listA := NewPackageList()
-	listA.Add(packages[0])
-	listA.Add(packages[1])
-	listA.Add(packages[2])
-	listA.Add(packages[3])
-	listA.Add(packages[6])
-
-	listB := NewPackageList()
-	listB.Add(packages[0])
-	listB.Add(packages[2])
-	listB.Add(packages[4])
-	listB.Add(packages[5])
-
-	reflistA := NewPackageRefListFromPackageList(listA)
-	reflistB := NewPackageRefListFromPackageList(listB)
-
-	diffAA, err := reflistA.Diff(reflistA, coll)
-	c.Check(err, IsNil)
-	c.Check(diffAA, HasLen, 0)
-
-	diffAB, err := reflistA.Diff(reflistB, coll)
-	c.Check(err, IsNil)
-	c.Check(diffAB, HasLen, 4)
-
-	c.Check(diffAB[0].Left, IsNil)
-	c.Check(diffAB[0].Right.String(), Equals, "app-1.1~bp2_amd64")
-
-	c.Check(diffAB[1].Left.String(), Equals, "app-1.1~bp1_i386")
-	c.Check(diffAB[1].Right.String(), Equals, "app-1.1~bp2_i386")
-
-	c.Check(diffAB[2].Left.String(), Equals, "dpkg-1.7_i386")
-	c.Check(diffAB[2].Right, IsNil)
-
-	c.Check(diffAB[3].Left.String(), Equals, "xyz-3.0_sparc")
-	c.Check(diffAB[3].Right, IsNil)
-
-	diffBA, err := reflistB.Diff(reflistA, coll)
-	c.Check(err, IsNil)
-	c.Check(diffBA, HasLen, 4)
-
-	c.Check(diffBA[0].Right, IsNil)
-	c.Check(diffBA[0].Left.String(), Equals, "app-1.1~bp2_amd64")
-
-	c.Check(diffBA[1].Right.String(), Equals, "app-1.1~bp1_i386")
-	c.Check(diffBA[1].Left.String(), Equals, "app-1.1~bp2_i386")
-
-	c.Check(diffBA[2].Right.String(), Equals, "dpkg-1.7_i386")
-	c.Check(diffBA[2].Left, IsNil)
-
-	c.Check(diffBA[3].Right.String(), Equals, "xyz-3.0_sparc")
-	c.Check(diffBA[3].Left, IsNil)
-
-}
-
-func (s *PackageListSuite) TestMerge(c *C) {
-	db, _ := database.OpenDB(c.MkDir())
-	coll := NewPackageCollection(db)
-
-	packages := []*Package{
-		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
-		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
-		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
-		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
-		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
-		&Package{Name: "dpkg", Version: "1.0", Architecture: "i386"},     //6
-		&Package{Name: "xyz", Version: "1.0", Architecture: "sparc"},     //7
-	}
-
-	for _, p := range packages {
-		coll.Update(p)
-	}
-
-	listA := NewPackageList()
-	listA.Add(packages[0])
-	listA.Add(packages[1])
-	listA.Add(packages[2])
-	listA.Add(packages[3])
-	listA.Add(packages[7])
-
-	listB := NewPackageList()
-	listB.Add(packages[0])
-	listB.Add(packages[2])
-	listB.Add(packages[4])
-	listB.Add(packages[5])
-	listB.Add(packages[6])
-
-	reflistA := NewPackageRefListFromPackageList(listA)
-	reflistB := NewPackageRefListFromPackageList(listB)
-
-	mergeAB := reflistA.Merge(reflistB)
-	mergeBA := reflistB.Merge(reflistA)
-
-	toStrSlice := func(reflist *PackageRefList) (result []string) {
-		result = make([]string, reflist.Len())
-		for i, r := range reflist.Refs {
-			result[i] = string(r)
-		}
-		return
-	}
-
-	c.Check(toStrSlice(mergeAB), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"})
-	c.Check(toStrSlice(mergeBA), DeepEquals,
-		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
-}

debian/local.go

@@ -0,0 +1,219 @@
+package debian
+
+import (
+	"bytes"
+	"code.google.com/p/go-uuid/uuid"
+	"fmt"
+	"github.com/smira/aptly/database"
+	"github.com/ugorji/go/codec"
+	"log"
+)
+
+// LocalRepo is a collection of packages created locally
+type LocalRepo struct {
+	// Permanent internal ID
+	UUID string
+	// User-assigned name
+	Name string
+	// Comment
+	Comment string
+	// "Snapshot" of current list of packages
+	packageRefs *PackageRefList
+}
+
+// NewLocalRepo creates new instance of Debian local repository
+func NewLocalRepo(name string, comment string) *LocalRepo {
+	return &LocalRepo{
+		UUID:    uuid.New(),
+		Name:    name,
+		Comment: comment,
+	}
+}
+
+// String interface
+func (repo *LocalRepo) String() string {
+	if repo.Comment != "" {
+		return fmt.Sprintf("[%s]: %s", repo.Name, repo.Comment)
+	}
+	return fmt.Sprintf("[%s]", repo.Name)
+}
+
+// NumPackages return number of packages in local repo
+func (repo *LocalRepo) NumPackages() int {
+	if repo.packageRefs == nil {
+		return 0
+	}
+	return repo.packageRefs.Len()
+}
+
+// RefList returns package list for repo
+func (repo *LocalRepo) RefList() *PackageRefList {
+	return repo.packageRefs
+}
+
+// UpdateRefList changes package list for local repo
+func (repo *LocalRepo) UpdateRefList(reflist *PackageRefList) {
+	repo.packageRefs = reflist
+}
+
+// Encode does msgpack encoding of LocalRepo
+func (repo *LocalRepo) Encode() []byte {
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(repo)
+
+	return buf.Bytes()
+}
+
+// Decode decodes msgpack representation into LocalRepo
+func (repo *LocalRepo) Decode(input []byte) error {
+	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+	return decoder.Decode(repo)
+}
+
+// Key is a unique id in DB
+func (repo *LocalRepo) Key() []byte {
+	return []byte("L" + repo.UUID)
+}
+
+// RefKey is a unique id for package reference list
+func (repo *LocalRepo) RefKey() []byte {
+	return []byte("E" + repo.UUID)
+}
+
+// LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
+type LocalRepoCollection struct {
+	db   database.Storage
+	list []*LocalRepo
+}
+
+// NewLocalRepoCollection loads LocalRepos from DB and makes up collection
+func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
+	result := &LocalRepoCollection{
+		db: db,
+	}
+
+	blobs := db.FetchByPrefix([]byte("L"))
+	result.list = make([]*LocalRepo, 0, len(blobs))
+
+	for _, blob := range blobs {
+		r := &LocalRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding mirror: %s\n", err)
+		} else {
+			result.list = append(result.list, r)
+		}
+	}
+
+	return result
+}
+
+// Add appends new repo to collection and saves it
+func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
+	for _, r := range collection.list {
+		if r.Name == repo.Name {
+			return fmt.Errorf("local repo with name %s already exists", repo.Name)
+		}
+	}
+
+	err := collection.Update(repo)
+	if err != nil {
+		return err
+	}
+
+	collection.list = append(collection.list, repo)
+	return nil
+}
+
+// Update stores updated information about repo in DB
+func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
+	err := collection.db.Put(repo.Key(), repo.Encode())
+	if err != nil {
+		return err
+	}
+	if repo.packageRefs != nil {
+		err = collection.db.Put(repo.RefKey(), repo.packageRefs.Encode())
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// LoadComplete loads additional information for local repo
+func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
+	encoded, err := collection.db.Get(repo.RefKey())
+	if err == database.ErrNotFound {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	repo.packageRefs = &PackageRefList{}
+	return repo.packageRefs.Decode(encoded)
+}
+
+// ByName looks up repository by name
+func (collection *LocalRepoCollection) ByName(name string) (*LocalRepo, error) {
+	for _, r := range collection.list {
+		if r.Name == name {
+			return r, nil
+		}
+	}
+	return nil, fmt.Errorf("local repo with name %s not found", name)
+}
+
+// ByUUID looks up repository by uuid
+func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
+	for _, r := range collection.list {
+		if r.UUID == uuid {
+			return r, nil
+		}
+	}
+	return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+}
+
+// ForEach runs method for each repository
+func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
+	var err error
+	for _, r := range collection.list {
+		err = handler(r)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// Len returns number of remote repos
+func (collection *LocalRepoCollection) Len() int {
+	return len(collection.list)
+}
+
+// Drop removes remote repo from collection
+func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
+	repoPosition := -1
+
+	for i, r := range collection.list {
+		if r == repo {
+			repoPosition = i
+			break
+		}
+	}
+
+	if repoPosition == -1 {
+		panic("local repo not found!")
+	}
+
+	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+
+	err := collection.db.Delete(repo.Key())
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Delete(repo.RefKey())
+}

debian/local_test.go

@@ -0,0 +1,195 @@
+package debian
+
+import (
+	"errors"
+	"github.com/smira/aptly/database"
+	. "launchpad.net/gocheck"
+)
+
+type LocalRepoSuite struct {
+	db      database.Storage
+	list    *PackageList
+	reflist *PackageRefList
+	repo    *LocalRepo
+}
+
+var _ = Suite(&LocalRepoSuite{})
+
+func (s *LocalRepoSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.list = NewPackageList()
+	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
+	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
+
+	s.repo = NewLocalRepo("lrepo", "Super repo")
+	s.repo.packageRefs = s.reflist
+}
+
+func (s *LocalRepoSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *LocalRepoSuite) TestString(c *C) {
+	c.Check(NewLocalRepo("lrepo", "My first repo").String(), Equals, "[lrepo]: My first repo")
+	c.Check(NewLocalRepo("lrepo2", "").String(), Equals, "[lrepo2]")
+}
+
+func (s *LocalRepoSuite) TestNumPackages(c *C) {
+	c.Check(NewLocalRepo("lrepo", "My first repo").NumPackages(), Equals, 0)
+	c.Check(s.repo.NumPackages(), Equals, 2)
+}
+
+func (s *LocalRepoSuite) TestRefList(c *C) {
+	c.Check(NewLocalRepo("lrepo", "My first repo").RefList(), IsNil)
+	c.Check(s.repo.RefList(), Equals, s.reflist)
+}
+
+func (s *LocalRepoSuite) TestUpdateRefList(c *C) {
+	s.repo.UpdateRefList(nil)
+	c.Check(s.repo.RefList(), IsNil)
+}
+
+func (s *LocalRepoSuite) TestEncodeDecode(c *C) {
+	repo := &LocalRepo{}
+	err := repo.Decode(s.repo.Encode())
+	c.Assert(err, IsNil)
+
+	c.Check(repo.Name, Equals, s.repo.Name)
+	c.Check(repo.Comment, Equals, s.repo.Comment)
+}
+
+func (s *LocalRepoSuite) TestKey(c *C) {
+	c.Assert(len(s.repo.Key()), Equals, 37)
+	c.Assert(s.repo.Key()[0], Equals, byte('L'))
+}
+
+func (s *LocalRepoSuite) TestRefKey(c *C) {
+	c.Assert(len(s.repo.RefKey()), Equals, 37)
+	c.Assert(s.repo.RefKey()[0], Equals, byte('E'))
+	c.Assert(s.repo.RefKey()[1:], DeepEquals, s.repo.Key()[1:])
+}
+
+type LocalRepoCollectionSuite struct {
+	db         database.Storage
+	collection *LocalRepoCollection
+	list       *PackageList
+	reflist    *PackageRefList
+}
+
+var _ = Suite(&LocalRepoCollectionSuite{})
+
+func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.collection = NewLocalRepoCollection(s.db)
+
+	s.list = NewPackageList()
+	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
+	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
+
+	s.reflist = NewPackageRefListFromPackageList(s.list)
+}
+
+func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
+	r, err := s.collection.ByName("local1")
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Add(repo), IsNil)
+	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
+
+	r, err = s.collection.ByName("local1")
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, repo.String())
+
+	collection := NewLocalRepoCollection(s.db)
+	r, err = collection.ByName("local1")
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, repo.String())
+}
+
+func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
+	r, err := s.collection.ByUUID("some-uuid")
+	c.Assert(err, ErrorMatches, "*.not found")
+
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Add(repo), IsNil)
+
+	r, err = s.collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
+	c.Assert(r.String(), Equals, repo.String())
+}
+
+func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	collection := NewLocalRepoCollection(s.db)
+	r, err := collection.ByName("local1")
+	c.Assert(err, IsNil)
+	c.Assert(r.packageRefs, IsNil)
+
+	repo.packageRefs = s.reflist
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	collection = NewLocalRepoCollection(s.db)
+	r, err = collection.ByName("local1")
+	c.Assert(err, IsNil)
+	c.Assert(r.packageRefs, IsNil)
+	c.Assert(r.NumPackages(), Equals, 0)
+	c.Assert(s.collection.LoadComplete(r), IsNil)
+	c.Assert(r.NumPackages(), Equals, 2)
+}
+
+func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
+	repo := NewLocalRepo("local1", "Comment 1")
+	s.collection.Add(repo)
+
+	count := 0
+	err := s.collection.ForEach(func(*LocalRepo) error {
+		count++
+		return nil
+	})
+	c.Assert(count, Equals, 1)
+	c.Assert(err, IsNil)
+
+	c.Check(s.collection.Len(), Equals, 1)
+
+	e := errors.New("c")
+
+	err = s.collection.ForEach(func(*LocalRepo) error {
+		return e
+	})
+	c.Assert(err, Equals, e)
+}
+
+func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
+	repo1 := NewLocalRepo("local1", "Comment 1")
+	s.collection.Add(repo1)
+
+	repo2 := NewLocalRepo("local2", "Comment 2")
+	s.collection.Add(repo2)
+
+	r1, _ := s.collection.ByUUID(repo1.UUID)
+	c.Check(r1, Equals, repo1)
+
+	err := s.collection.Drop(repo1)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByUUID(repo1.UUID)
+	c.Check(err, ErrorMatches, "local repo .* not found")
+
+	collection := NewLocalRepoCollection(s.db)
+	_, err = collection.ByName("local1")
+	c.Check(err, ErrorMatches, "local repo .* not found")
+
+	r2, _ := collection.ByName("local2")
+	c.Check(r2.String(), Equals, repo2.String())
+
+	c.Check(func() { s.collection.Drop(repo1) }, Panics, "local repo not found!")
+}

debian/package.go

@@ -1,71 +1,37 @@
 package debian
 
 import (
-	"bytes"
 	"fmt"
-	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
-	"github.com/ugorji/go/codec"
-	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 )
 
-// PackageFile is a single file entry in package
-type PackageFile struct {
-	Filename  string
-	Checksums utils.ChecksumInfo
-}
-
-// Verify that package file is present and correct
-func (f *PackageFile) Verify(packageRepo *Repository) (bool, error) {
-	poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
-	if err != nil {
-		return false, err
-	}
-
-	st, err := os.Stat(poolPath)
-	if err != nil {
-		return false, nil
-	}
-
-	// verify size
-	// TODO: verify checksum if configured
-	return st.Size() == f.Checksums.Size, nil
-}
-
 // Package is single instance of Debian package
 type Package struct {
+	// Basic package properties
 	Name         string
 	Version      string
 	Architecture string
-	Source       string
-	Provides     []string
-	// Various dependencies
-	Depends    []string
-	PreDepends []string
-	Suggests   []string
-	Recommends []string
-	// Files in package
-	Files []PackageFile
-	// Extra information from stanza
-	Extra Stanza
-}
-
-func parseDependencies(input Stanza, key string) []string {
-	value, ok := input[key]
-	if !ok {
-		return nil
-	}
-
-	delete(input, key)
-
-	result := strings.Split(value, ",")
-	for i := range result {
-		result[i] = strings.TrimSpace(result[i])
-	}
-	return result
+	// If this source package, this field holds "real" architecture value,
+	// while Architecture would be equal to "source"
+	SourceArchitecture string
+	// For binary package, name of source package
+	Source string
+	// List of virtual packages this package provides
+	Provides []string
+	// Is this source package
+	IsSource bool
+	// Hash of files section
+	FilesHash uint64
+	// Offload fields
+	deps  *PackageDependencies
+	extra *Stanza
+	files *PackageFiles
+	// Mother collection
+	collection *PackageCollection
 }
 
 // NewPackageFromControlFile creates Package from parsed Debian control file
@@ -75,7 +41,6 @@ func NewPackageFromControlFile(input Stanza) *Package {
 		Version:      input["Version"],
 		Architecture: input["Architecture"],
 		Source:       input["Source"],
-		Files:        make([]PackageFile, 0, 1),
 	}
 
 	delete(input, "Package")
@@ -85,15 +50,16 @@ func NewPackageFromControlFile(input Stanza) *Package {
 
 	filesize, _ := strconv.ParseInt(input["Size"], 10, 64)
 
-	result.Files = append(result.Files, PackageFile{
-		Filename: input["Filename"],
+	result.UpdateFiles(PackageFiles{PackageFile{
+		Filename:     filepath.Base(input["Filename"]),
+		downloadPath: filepath.Dir(input["Filename"]),
 		Checksums: utils.ChecksumInfo{
 			Size:   filesize,
-			MD5:    input["MD5sum"],
-			SHA1:   input["SHA1"],
-			SHA256: input["SHA256"],
+			MD5:    strings.TrimSpace(input["MD5sum"]),
+			SHA1:   strings.TrimSpace(input["SHA1"]),
+			SHA256: strings.TrimSpace(input["SHA256"]),
 		},
-	})
+	}})
 
 	delete(input, "Filename")
 	delete(input, "MD5sum")
@@ -101,153 +67,344 @@ func NewPackageFromControlFile(input Stanza) *Package {
 	delete(input, "SHA256")
 	delete(input, "Size")
 
-	result.Depends = parseDependencies(input, "Depends")
-	result.PreDepends = parseDependencies(input, "Pre-Depends")
-	result.Suggests = parseDependencies(input, "Suggests")
-	result.Recommends = parseDependencies(input, "Recommends")
+	depends := &PackageDependencies{}
+	depends.Depends = parseDependencies(input, "Depends")
+	depends.PreDepends = parseDependencies(input, "Pre-Depends")
+	depends.Suggests = parseDependencies(input, "Suggests")
+	depends.Recommends = parseDependencies(input, "Recommends")
+	result.deps = depends
+
 	result.Provides = parseDependencies(input, "Provides")
 
-	result.Extra = input
+	result.extra = &input
 
 	return result
 }
 
+// NewSourcePackageFromControlFile creates Package from parsed Debian control file for source package
+func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
+	result := &Package{
+		IsSource:           true,
+		Name:               input["Package"],
+		Version:            input["Version"],
+		Architecture:       "source",
+		SourceArchitecture: input["Architecture"],
+	}
+
+	delete(input, "Package")
+	delete(input, "Version")
+	delete(input, "Architecture")
+
+	files := make(PackageFiles, 0, 3)
+
+	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
+		for _, line := range strings.Split(input[field], "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" {
+				continue
+			}
+			parts := strings.Fields(line)
+
+			if len(parts) != 3 {
+				return fmt.Errorf("unparseable hash sum line: %#v", line)
+			}
+
+			size, err := strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return fmt.Errorf("unable to parse size: %s", err)
+			}
+
+			filename := filepath.Base(parts[2])
+
+			found := false
+			pos := 0
+			for i, file := range files {
+				if file.Filename == filename {
+					found = true
+					pos = i
+					break
+				}
+			}
+
+			if !found {
+				files = append(files, PackageFile{Filename: filename, downloadPath: input["Directory"]})
+				pos = len(files) - 1
+			}
+
+			files[pos].Checksums.Size = size
+			setter(&files[pos].Checksums, parts[0])
+		}
+
+		delete(input, field)
+
+		return nil
+	}
+
+	err := parseSums("Files", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return nil, err
+	}
+	err = parseSums("Checksums-Sha1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return nil, err
+	}
+	err = parseSums("Checksums-Sha256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return nil, err
+	}
+
+	result.UpdateFiles(files)
+
+	depends := &PackageDependencies{}
+	depends.BuildDepends = parseDependencies(input, "Build-Depends")
+	depends.BuildDependsInDep = parseDependencies(input, "Build-Depends-Indep")
+	result.deps = depends
+
+	result.extra = &input
+
+	return result, nil
+}
+
 // Key returns unique key identifying package
-func (p *Package) Key() []byte {
-	return []byte("P" + p.Architecture + " " + p.Name + " " + p.Version)
-}
-
-// Encode does msgpack encoding of Package
-func (p *Package) Encode() []byte {
-	var buf bytes.Buffer
-
-	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(p)
-
-	return buf.Bytes()
-}
-
-// Decode decodes msgpack representation into Package
-func (p *Package) Decode(input []byte) error {
-	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
-	return decoder.Decode(p)
+func (p *Package) Key(prefix string) []byte {
+	return []byte(prefix + "P" + p.Architecture + " " + p.Name + " " + p.Version)
 }
 
 // String creates readable representation
 func (p *Package) String() string {
-	return fmt.Sprintf("%s-%s_%s", p.Name, p.Version, p.Architecture)
+	return fmt.Sprintf("%s_%s_%s", p.Name, p.Version, p.Architecture)
 }
 
 // MatchesArchitecture checks whether packages matches specified architecture
 func (p *Package) MatchesArchitecture(arch string) bool {
-	if p.Architecture == "all" {
+	if p.Architecture == "all" && arch != "source" {
 		return true
 	}
 
 	return p.Architecture == arch
 }
 
+// MatchesDependency checks whether package matches specified dependency
+func (p *Package) MatchesDependency(dep Dependency) bool {
+	if dep.Pkg != p.Name {
+		return false
+	}
+
+	if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
+		return false
+	}
+
+	if dep.Relation == VersionDontCare {
+		return true
+	}
+
+	r := CompareVersions(p.Version, dep.Version)
+	switch dep.Relation {
+	case VersionEqual:
+		return r == 0
+	case VersionLess:
+		return r < 0
+	case VersionGreater:
+		return r > 0
+	case VersionLessOrEqual:
+		return r <= 0
+	case VersionGreaterOrEqual:
+		return r >= 0
+	}
+
+	panic("unknown relation")
+}
+
 // GetDependencies compiles list of dependenices by flags from options
 func (p *Package) GetDependencies(options int) (dependencies []string) {
+	deps := p.Deps()
+
 	dependencies = make([]string, 0, 30)
-	dependencies = append(dependencies, p.Depends...)
-	dependencies = append(dependencies, p.PreDepends...)
+	dependencies = append(dependencies, deps.Depends...)
+	dependencies = append(dependencies, deps.PreDepends...)
 
 	if options&DepFollowRecommends == DepFollowRecommends {
-		dependencies = append(dependencies, p.Recommends...)
+		dependencies = append(dependencies, deps.Recommends...)
 	}
 
 	if options&DepFollowSuggests == DepFollowSuggests {
-		dependencies = append(dependencies, p.Suggests...)
+		dependencies = append(dependencies, deps.Suggests...)
+	}
+
+	if options&DepFollowBuild == DepFollowBuild {
+		dependencies = append(dependencies, deps.BuildDepends...)
+		dependencies = append(dependencies, deps.BuildDependsInDep...)
+	}
+
+	if options&DepFollowSource == DepFollowSource {
+		source := p.Source
+		if source == "" {
+			source = p.Name
+		}
+		if strings.Index(source, ")") != -1 {
+			dependencies = append(dependencies, fmt.Sprintf("%s {source}", source))
+		} else {
+			dependencies = append(dependencies, fmt.Sprintf("%s (= %s) {source}", source, p.Version))
+		}
 	}
 
 	return
 }
 
+// Extra returns Stanza of extra fields (it may load it from collection)
+func (p *Package) Extra() Stanza {
+	if p.extra == nil {
+		if p.collection == nil {
+			panic("extra == nil && collection == nil")
+		}
+		p.extra = p.collection.loadExtra(p)
+	}
+
+	return *p.extra
+}
+
+// Deps returns parsed package dependencies (it may load it from collection)
+func (p *Package) Deps() *PackageDependencies {
+	if p.deps == nil {
+		if p.collection == nil {
+			panic("deps == nil && collection == nil")
+		}
+
+		p.deps = p.collection.loadDependencies(p)
+	}
+
+	return p.deps
+}
+
+// Files returns parsed files records (it may load it from collection)
+func (p *Package) Files() PackageFiles {
+	if p.files == nil {
+		if p.collection == nil {
+			panic("files == nil && collection == nil")
+		}
+
+		p.files = p.collection.loadFiles(p)
+	}
+
+	return *p.files
+}
+
+// UpdateFiles saves new state of files
+func (p *Package) UpdateFiles(files PackageFiles) {
+	p.files = &files
+	p.FilesHash = files.Hash()
+}
+
 // Stanza creates original stanza from package
 func (p *Package) Stanza() (result Stanza) {
-	result = p.Extra.Copy()
+	result = p.Extra().Copy()
 	result["Package"] = p.Name
 	result["Version"] = p.Version
-	result["Filename"] = p.Files[0].Filename
-	result["Architecture"] = p.Architecture
-	result["Source"] = p.Source
 
-	if p.Files[0].Checksums.MD5 != "" {
-		result["MD5sum"] = p.Files[0].Checksums.MD5
-	}
-	if p.Files[0].Checksums.SHA1 != "" {
-		result["SHA1"] = p.Files[0].Checksums.SHA1
-	}
-	if p.Files[0].Checksums.SHA256 != "" {
-		result["SHA256"] = p.Files[0].Checksums.SHA256
+	if p.IsSource {
+		result["Architecture"] = p.SourceArchitecture
+	} else {
+		result["Architecture"] = p.Architecture
+		result["Source"] = p.Source
 	}
 
-	if p.Depends != nil {
-		result["Depends"] = strings.Join(p.Depends, ", ")
+	if p.IsSource {
+		md5, sha1, sha256 := make([]string, 0), make([]string, 0), make([]string, 0)
+
+		for _, f := range p.Files() {
+			if f.Checksums.MD5 != "" {
+				md5 = append(md5, fmt.Sprintf(" %s %d %s\n", f.Checksums.MD5, f.Checksums.Size, f.Filename))
+			}
+			if f.Checksums.SHA1 != "" {
+				sha1 = append(sha1, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA1, f.Checksums.Size, f.Filename))
+			}
+			if f.Checksums.SHA256 != "" {
+				sha256 = append(sha256, fmt.Sprintf(" %s %d %s\n", f.Checksums.SHA256, f.Checksums.Size, f.Filename))
+			}
+		}
+
+		result["Files"] = strings.Join(md5, "")
+		result["Checksums-Sha1"] = strings.Join(sha1, "")
+		result["Checksums-Sha256"] = strings.Join(sha256, "")
+	} else {
+		f := p.Files()[0]
+		result["Filename"] = f.DownloadURL()
+		if f.Checksums.MD5 != "" {
+			result["MD5sum"] = f.Checksums.MD5
+		}
+		if f.Checksums.SHA1 != "" {
+			result["SHA1"] = " " + f.Checksums.SHA1
+		}
+		if f.Checksums.SHA256 != "" {
+			result["SHA256"] = " " + f.Checksums.SHA256
+		}
+		result["Size"] = fmt.Sprintf("%d", f.Checksums.Size)
 	}
-	if p.PreDepends != nil {
-		result["Pre-Depends"] = strings.Join(p.PreDepends, ", ")
+
+	deps := p.Deps()
+
+	if deps.Depends != nil {
+		result["Depends"] = strings.Join(deps.Depends, ", ")
 	}
-	if p.Suggests != nil {
-		result["Suggests"] = strings.Join(p.Suggests, ", ")
+	if deps.PreDepends != nil {
+		result["Pre-Depends"] = strings.Join(deps.PreDepends, ", ")
 	}
-	if p.Recommends != nil {
-		result["Recommends"] = strings.Join(p.Recommends, ", ")
+	if deps.Suggests != nil {
+		result["Suggests"] = strings.Join(deps.Suggests, ", ")
+	}
+	if deps.Recommends != nil {
+		result["Recommends"] = strings.Join(deps.Recommends, ", ")
 	}
 	if p.Provides != nil {
 		result["Provides"] = strings.Join(p.Provides, ", ")
 	}
-
-	result["Size"] = fmt.Sprintf("%d", p.Files[0].Checksums.Size)
+	if deps.BuildDepends != nil {
+		result["Build-Depends"] = strings.Join(deps.BuildDepends, ", ")
+	}
+	if deps.BuildDependsInDep != nil {
+		result["Build-Depends-Indep"] = strings.Join(deps.BuildDependsInDep, ", ")
+	}
 
 	return
 }
 
 // Equals compares two packages to be identical
 func (p *Package) Equals(p2 *Package) bool {
-	if len(p.Files) != len(p2.Files) {
-		return false
-	}
-
-	for i, f := range p.Files {
-		if p2.Files[i] != f {
-			return false
-		}
-	}
-
-	return p.Name == p2.Name && p.Version == p2.Version &&
-		p.Architecture == p2.Architecture && utils.StrSlicesEqual(p.Depends, p2.Depends) &&
-		utils.StrSlicesEqual(p.PreDepends, p2.PreDepends) && utils.StrSlicesEqual(p.Suggests, p2.Suggests) &&
-		utils.StrSlicesEqual(p.Recommends, p2.Recommends) && utils.StrMapsEqual(p.Extra, p2.Extra) &&
-		p.Source == p2.Source && utils.StrSlicesEqual(p.Provides, p2.Provides)
+	return p.Name == p2.Name && p.Version == p2.Version && p.SourceArchitecture == p2.SourceArchitecture &&
+		p.Architecture == p2.Architecture && p.Source == p2.Source && p.IsSource == p2.IsSource &&
+		p.FilesHash == p2.FilesHash
 }
 
 // LinkFromPool links package file from pool to dist's pool location
-func (p *Package) LinkFromPool(packageRepo *Repository, prefix string, component string) error {
+func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, prefix string, component string) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
 	}
 
-	for i, f := range p.Files {
-		sourcePath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
+	for i, f := range p.Files() {
+		sourcePath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
 		if err != nil {
 			return err
 		}
 
-		relPath, err := packageRepo.LinkFromPool(prefix, component, sourcePath, poolDir)
+		relPath, err := publishedStorage.LinkFromPool(prefix, component, poolDir, packagePool, sourcePath)
 		if err != nil {
 			return err
 		}
 
-		p.Files[i].Filename = relPath
+		dir := filepath.Dir(relPath)
+		if p.IsSource {
+			p.Extra()["Directory"] = dir
+		} else {
+			p.Files()[i].downloadPath = dir
+		}
 	}
 
 	return nil
 }
 
-// PoolDirectory returns directory in package pool for this package files
+// PoolDirectory returns directory in package pool of published repository for this package files
 func (p *Package) PoolDirectory() (string, error) {
 	source := p.Source
 	if source == "" {
@@ -273,27 +430,27 @@ func (p *Package) PoolDirectory() (string, error) {
 type PackageDownloadTask struct {
 	RepoURI         string
 	DestinationPath string
-	Size            int64
+	Checksums       utils.ChecksumInfo
 }
 
 // DownloadList returns list of missing package files for download in format
 // [[srcpath, dstpath]]
-func (p *Package) DownloadList(packageRepo *Repository) (result []PackageDownloadTask, err error) {
+func (p *Package) DownloadList(packagePool aptly.PackagePool) (result []PackageDownloadTask, err error) {
 	result = make([]PackageDownloadTask, 0, 1)
 
-	for _, f := range p.Files {
-		poolPath, err := packageRepo.PoolPath(f.Filename, f.Checksums.MD5)
+	for _, f := range p.Files() {
+		poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
 		if err != nil {
 			return nil, err
 		}
 
-		verified, err := f.Verify(packageRepo)
+		verified, err := f.Verify(packagePool)
 		if err != nil {
 			return nil, err
 		}
 
 		if !verified {
-			result = append(result, PackageDownloadTask{RepoURI: f.Filename, DestinationPath: poolPath, Size: f.Checksums.Size})
+			result = append(result, PackageDownloadTask{RepoURI: f.DownloadURL(), DestinationPath: poolPath, Checksums: f.Checksums})
 		}
 	}
 
@@ -301,11 +458,11 @@ func (p *Package) DownloadList(packageRepo *Repository) (result []PackageDownloa
 }
 
 // VerifyFiles verifies that all package files have neen correctly downloaded
-func (p *Package) VerifyFiles(packageRepo *Repository) (result bool, err error) {
+func (p *Package) VerifyFiles(packagePool aptly.PackagePool) (result bool, err error) {
 	result = true
 
-	for _, f := range p.Files {
-		result, err = f.Verify(packageRepo)
+	for _, f := range p.Files() {
+		result, err = f.Verify(packagePool)
 		if err != nil || !result {
 			return
 		}
@@ -314,34 +471,17 @@ func (p *Package) VerifyFiles(packageRepo *Repository) (result bool, err error)
 	return
 }
 
-// PackageCollection does management of packages in DB
-type PackageCollection struct {
-	db database.Storage
-}
+// FilepathList returns list of paths to files in package repository
+func (p *Package) FilepathList(packagePool aptly.PackagePool) ([]string, error) {
+	var err error
+	result := make([]string, len(p.Files()))
 
-// NewPackageCollection creates new PackageCollection and binds it to database
-func NewPackageCollection(db database.Storage) *PackageCollection {
-	return &PackageCollection{
-		db: db,
-	}
-}
-
-// ByKey find package in DB by its key
-func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
-	encoded, err := collection.db.Get(key)
-	if err != nil {
-		return nil, err
+	for i, f := range p.Files() {
+		result[i], err = packagePool.RelativePath(f.Filename, f.Checksums.MD5)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	p := &Package{}
-	err = p.Decode(encoded)
-	if err != nil {
-		return nil, err
-	}
-	return p, nil
-}
-
-// Update adds or updates information about package in DB
-func (collection *PackageCollection) Update(p *Package) error {
-	return collection.db.Put(p.Key(), p.Encode())
+	return result, nil
 }

debian/package_collection.go

@@ -0,0 +1,256 @@
+package debian
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/smira/aptly/database"
+	"github.com/ugorji/go/codec"
+	"path/filepath"
+)
+
+// PackageCollection does management of packages in DB
+type PackageCollection struct {
+	db           database.Storage
+	encodeBuffer bytes.Buffer
+}
+
+// NewPackageCollection creates new PackageCollection and binds it to database
+func NewPackageCollection(db database.Storage) *PackageCollection {
+	return &PackageCollection{
+		db: db,
+	}
+}
+
+// oldPackage is Package struct for aptly < 0.4 with all fields in one struct
+// It is used to decode old aptly DBs
+type oldPackage struct {
+	IsSource           bool
+	Name               string
+	Version            string
+	Architecture       string
+	SourceArchitecture string
+	Source             string
+	Provides           []string
+	Depends            []string
+	BuildDepends       []string
+	BuildDependsInDep  []string
+	PreDepends         []string
+	Suggests           []string
+	Recommends         []string
+	Files              []PackageFile
+	Extra              Stanza
+}
+
+// ByKey find package in DB by its key
+func (collection *PackageCollection) ByKey(key []byte) (*Package, error) {
+	encoded, err := collection.db.Get(key)
+	if err != nil {
+		return nil, err
+	}
+
+	p := &Package{}
+
+	if len(encoded) > 2 && (encoded[0] != 0xc1 || encoded[1] != 0x1) {
+		oldp := &oldPackage{}
+
+		decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+		err = decoder.Decode(oldp)
+		if err != nil {
+			return nil, err
+		}
+
+		p.Name = oldp.Name
+		p.Version = oldp.Version
+		p.Architecture = oldp.Architecture
+		p.IsSource = oldp.IsSource
+		p.SourceArchitecture = oldp.SourceArchitecture
+		p.Source = oldp.Source
+		p.Provides = oldp.Provides
+
+		p.deps = &PackageDependencies{
+			Depends:           oldp.Depends,
+			BuildDepends:      oldp.BuildDepends,
+			BuildDependsInDep: oldp.BuildDependsInDep,
+			PreDepends:        oldp.PreDepends,
+			Suggests:          oldp.Suggests,
+			Recommends:        oldp.Recommends,
+		}
+
+		p.extra = &oldp.Extra
+		for i := range oldp.Files {
+			oldp.Files[i].Filename = filepath.Base(oldp.Files[i].Filename)
+		}
+		p.UpdateFiles(PackageFiles(oldp.Files))
+
+		// Save in new format
+		err = collection.internalUpdate(p)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		decoder := codec.NewDecoderBytes(encoded[2:], &codec.MsgpackHandle{})
+		err = decoder.Decode(p)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	p.collection = collection
+
+	return p, nil
+}
+
+// loadExtra loads Stanza with all the xtra information about the package
+func (collection *PackageCollection) loadExtra(p *Package) *Stanza {
+	encoded, err := collection.db.Get(p.Key("xE"))
+	if err != nil {
+		panic("unable to load extra")
+	}
+
+	stanza := &Stanza{}
+
+	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	err = decoder.Decode(stanza)
+	if err != nil {
+		panic("unable to decode extra")
+	}
+
+	return stanza
+}
+
+// loadDependencies loads dependencies for the package
+func (collection *PackageCollection) loadDependencies(p *Package) *PackageDependencies {
+	encoded, err := collection.db.Get(p.Key("xD"))
+	if err != nil {
+		panic(fmt.Sprintf("unable to load deps: %s, %s", p, err))
+	}
+
+	deps := &PackageDependencies{}
+
+	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	err = decoder.Decode(deps)
+	if err != nil {
+		panic("unable to decode deps")
+	}
+
+	return deps
+}
+
+// loadFiles loads additional PackageFiles record
+func (collection *PackageCollection) loadFiles(p *Package) *PackageFiles {
+	encoded, err := collection.db.Get(p.Key("xF"))
+	if err != nil {
+		panic("unable to load files")
+	}
+
+	files := &PackageFiles{}
+
+	decoder := codec.NewDecoderBytes(encoded, &codec.MsgpackHandle{})
+	err = decoder.Decode(files)
+	if err != nil {
+		panic("unable to decode files")
+	}
+
+	return files
+}
+
+// Update adds or updates information about package in DB checking for conficts first
+func (collection *PackageCollection) Update(p *Package) error {
+	existing, err := collection.ByKey(p.Key(""))
+	if err == nil {
+		// if .Files is different, consider to be conflict
+		if p.FilesHash != existing.FilesHash {
+			return fmt.Errorf("unable to save: %s, conflict with existing packge", p)
+		}
+		// ok, .Files are the same, but maybe some meta-data is different, proceed to saving
+	} else {
+		if err != database.ErrNotFound {
+			return err
+		}
+		// ok, package doesn't exist yet
+	}
+
+	return collection.internalUpdate(p)
+}
+
+// internalUpdate updates information in DB about package and offloaded fields
+func (collection *PackageCollection) internalUpdate(p *Package) error {
+	encoder := codec.NewEncoder(&collection.encodeBuffer, &codec.MsgpackHandle{})
+
+	collection.encodeBuffer.Reset()
+	collection.encodeBuffer.WriteByte(0xc1)
+	collection.encodeBuffer.WriteByte(0x1)
+	err := encoder.Encode(p)
+	if err != nil {
+		return err
+	}
+
+	err = collection.db.Put(p.Key(""), collection.encodeBuffer.Bytes())
+	if err != nil {
+		return err
+	}
+
+	// Encode offloaded fields one by one
+	if p.files != nil {
+		collection.encodeBuffer.Reset()
+		err = encoder.Encode(*p.files)
+		if err != nil {
+			return err
+		}
+
+		err = collection.db.Put(p.Key("xF"), collection.encodeBuffer.Bytes())
+		if err != nil {
+			return err
+		}
+	}
+
+	if p.deps != nil {
+		collection.encodeBuffer.Reset()
+		err = encoder.Encode(*p.deps)
+		if err != nil {
+			return err
+		}
+
+		err = collection.db.Put(p.Key("xD"), collection.encodeBuffer.Bytes())
+		if err != nil {
+			return err
+		}
+
+		p.deps = nil
+	}
+
+	if p.extra != nil {
+		collection.encodeBuffer.Reset()
+		err = encoder.Encode(*p.extra)
+		if err != nil {
+			return err
+		}
+
+		err = collection.db.Put(p.Key("xE"), collection.encodeBuffer.Bytes())
+		if err != nil {
+			return err
+		}
+
+		p.extra = nil
+	}
+
+	p.collection = collection
+
+	return nil
+}
+
+// AllPackageRefs returns list of all packages as PackageRefList
+func (collection *PackageCollection) AllPackageRefs() *PackageRefList {
+	return &PackageRefList{Refs: collection.db.KeysByPrefix([]byte("P"))}
+}
+
+// DeleteByKey deletes package in DB by key
+func (collection *PackageCollection) DeleteByKey(key []byte) error {
+	for _, key := range [][]byte{key, append([]byte("xF"), key...), append([]byte("xD"), key...), append([]byte("xE"), key...)} {
+		err := collection.db.Delete(key)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

debian/package_collection_test.go

@@ -0,0 +1,205 @@
+package debian
+
+import (
+	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
+	. "launchpad.net/gocheck"
+)
+
+type PackageCollectionSuite struct {
+	collection *PackageCollection
+	p          *Package
+	db         database.Storage
+}
+
+var _ = Suite(&PackageCollectionSuite{})
+
+func (s *PackageCollectionSuite) SetUpTest(c *C) {
+	s.p = NewPackageFromControlFile(packageStanza.Copy())
+	s.db, _ = database.OpenDB(c.MkDir())
+	s.collection = NewPackageCollection(s.db)
+}
+
+func (s *PackageCollectionSuite) TearDownTest(c *C) {
+	s.db.Close()
+}
+
+func (s *PackageCollectionSuite) TestUpdate(c *C) {
+	// package doesn't exist, update ok
+	err := s.collection.Update(s.p)
+	c.Assert(err, IsNil)
+	res, err := s.collection.ByKey(s.p.Key(""))
+	c.Assert(err, IsNil)
+	c.Assert(res.Equals(s.p), Equals, true)
+
+	// same package, ok
+	p2 := NewPackageFromControlFile(packageStanza.Copy())
+	err = s.collection.Update(p2)
+	c.Assert(err, IsNil)
+	res, err = s.collection.ByKey(p2.Key(""))
+	c.Assert(err, IsNil)
+	c.Assert(res.Equals(s.p), Equals, true)
+
+	// change some metadata
+	p2.Source = "lala"
+	err = s.collection.Update(p2)
+	c.Assert(err, IsNil)
+	res, err = s.collection.ByKey(p2.Key(""))
+	c.Assert(err, IsNil)
+	c.Assert(res.Equals(s.p), Equals, false)
+	c.Assert(res.Equals(p2), Equals, true)
+
+	// change file info
+	p2 = NewPackageFromControlFile(packageStanza.Copy())
+	p2.UpdateFiles(nil)
+	res, err = s.collection.ByKey(p2.Key(""))
+	err = s.collection.Update(p2)
+	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
+	p2 = NewPackageFromControlFile(packageStanza.Copy())
+	files := p2.Files()
+	files[0].Checksums.MD5 = "abcdef"
+	p2.UpdateFiles(files)
+	res, err = s.collection.ByKey(p2.Key(""))
+	err = s.collection.Update(p2)
+	c.Assert(err, ErrorMatches, ".*conflict with existing packge")
+}
+
+func (s *PackageCollectionSuite) TestByKey(c *C) {
+	err := s.collection.Update(s.p)
+	c.Assert(err, IsNil)
+
+	p2, err := s.collection.ByKey(s.p.Key(""))
+	c.Assert(err, IsNil)
+	c.Assert(p2.Equals(s.p), Equals, true)
+
+	c.Check(p2.GetDependencies(0), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)"})
+	c.Check(p2.Extra()["Priority"], Equals, "extra")
+	c.Check(p2.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
+}
+
+func (s *PackageCollectionSuite) TestByKeyOld_0_3(c *C) {
+	key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1")
+	s.db.Put(key, old_0_3_Package)
+
+	p, err := s.collection.ByKey(key)
+	c.Check(err, IsNil)
+	c.Check(p.Name, Equals, "vmware-view-open-client")
+	c.Check(p.Version, Equals, "4.5.0-297975+dfsg-4+b1")
+	c.Check(p.Architecture, Equals, "i386")
+	c.Check(p.Files(), DeepEquals, PackageFiles{
+		PackageFile{Filename: "vmware-view-open-client_4.5.0-297975+dfsg-4+b1_i386.deb",
+			Checksums: utils.ChecksumInfo{
+				Size:   520080,
+				MD5:    "9c61b54e2638a18f955a695b9162d6af",
+				SHA1:   "5b7c99e64a70f4f509bfa3a674088ff9cef68163",
+				SHA256: "4a9e4b2d9b3db13f9a29e522f3ffbb34eee96fc6f34a0647042ab1b5b0f2e04d"}}})
+	c.Check(p.GetDependencies(0), DeepEquals, []string{"libatk1.0-0 (>= 1.12.4)", "libboost-signals1.49.0 (>= 1.49.0-1)",
+		"libc6 (>= 2.3.6-6~)", "libcairo2 (>= 1.2.4)", "libcurl3 (>= 7.18.0)", "libfontconfig1 (>= 2.8.0)", "libfreetype6 (>= 2.2.1)",
+		"libgcc1 (>= 1:4.1.1)", "libgdk-pixbuf2.0-0 (>= 2.22.0)", "libglib2.0-0 (>= 2.24.0)", "libgtk2.0-0 (>= 2.24.0)",
+		"libicu48 (>= 4.8-1)", "libpango1.0-0 (>= 1.14.0)", "libssl1.0.0 (>= 1.0.0)", "libstdc++6 (>= 4.6)", "libx11-6",
+		"libxml2 (>= 2.7.4)", "rdesktop"})
+	c.Check(p.Extra()["Priority"], Equals, "optional")
+}
+
+func (s *PackageCollectionSuite) TestAllPackageRefs(c *C) {
+	err := s.collection.Update(s.p)
+	c.Assert(err, IsNil)
+
+	refs := s.collection.AllPackageRefs()
+	c.Check(refs.Len(), Equals, 1)
+	c.Check(refs.Refs[0], DeepEquals, s.p.Key(""))
+}
+
+func (s *PackageCollectionSuite) TestDeleteByKey(c *C) {
+	err := s.collection.Update(s.p)
+	c.Assert(err, IsNil)
+
+	_, err = s.db.Get(s.p.Key(""))
+	c.Check(err, IsNil)
+
+	_, err = s.db.Get(s.p.Key("xD"))
+	c.Check(err, IsNil)
+
+	_, err = s.db.Get(s.p.Key("xE"))
+	c.Check(err, IsNil)
+
+	_, err = s.db.Get(s.p.Key("xF"))
+	c.Check(err, IsNil)
+
+	err = s.collection.DeleteByKey(s.p.Key(""))
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByKey(s.p.Key(""))
+	c.Check(err, ErrorMatches, "key not found")
+
+	_, err = s.db.Get(s.p.Key(""))
+	c.Check(err, ErrorMatches, "key not found")
+
+	_, err = s.db.Get(s.p.Key("xD"))
+	c.Check(err, ErrorMatches, "key not found")
+
+	_, err = s.db.Get(s.p.Key("xE"))
+	c.Check(err, ErrorMatches, "key not found")
+
+	_, err = s.db.Get(s.p.Key("xF"))
+	c.Check(err, ErrorMatches, "key not found")
+}
+
+// This is old package (pre-0.4) that would habe to be converted
+var old_0_3_Package = []byte{0x8f, 0xac, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74, 0x75, 0x72, 0x65, 0xa4, 0x69, 0x33, 0x38, 0x36,
+	0xac, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x73, 0xc0, 0xb1, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x44, 0x65,
+	0x70, 0x65, 0x6e, 0x64, 0x73, 0x49, 0x6e, 0x44, 0x65, 0x70, 0xc0, 0xa7, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x73, 0xdc, 0x0, 0x12,
+	0xb7, 0x6c, 0x69, 0x62, 0x61, 0x74, 0x6b, 0x31, 0x2e, 0x30, 0x2d, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x31, 0x32, 0x2e,
+	0x34, 0x29, 0xda, 0x0, 0x24, 0x6c, 0x69, 0x62, 0x62, 0x6f, 0x6f, 0x73, 0x74, 0x2d, 0x73, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x73, 0x31,
+	0x2e, 0x34, 0x39, 0x2e, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x34, 0x39, 0x2e, 0x30, 0x2d, 0x31, 0x29, 0xb3, 0x6c, 0x69,
+	0x62, 0x63, 0x36, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x33, 0x2e, 0x36, 0x2d, 0x36, 0x7e, 0x29, 0xb4, 0x6c, 0x69, 0x62, 0x63,
+	0x61, 0x69, 0x72, 0x6f, 0x32, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x32, 0x2e, 0x34, 0x29, 0xb4, 0x6c, 0x69, 0x62, 0x63, 0x75,
+	0x72, 0x6c, 0x33, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x37, 0x2e, 0x31, 0x38, 0x2e, 0x30, 0x29, 0xb9, 0x6c, 0x69, 0x62, 0x66, 0x6f, 0x6e,
+	0x74, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x31, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x38, 0x2e, 0x30, 0x29, 0xb7, 0x6c, 0x69,
+	0x62, 0x66, 0x72, 0x65, 0x65, 0x74, 0x79, 0x70, 0x65, 0x36, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x32, 0x2e, 0x31, 0x29, 0xb4,
+	0x6c, 0x69, 0x62, 0x67, 0x63, 0x63, 0x31, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x3a, 0x34, 0x2e, 0x31, 0x2e, 0x31, 0x29, 0xbe, 0x6c,
+	0x69, 0x62, 0x67, 0x64, 0x6b, 0x2d, 0x70, 0x69, 0x78, 0x62, 0x75, 0x66, 0x32, 0x2e, 0x30, 0x2d, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20,
+	0x32, 0x2e, 0x32, 0x32, 0x2e, 0x30, 0x29, 0xb8, 0x6c, 0x69, 0x62, 0x67, 0x6c, 0x69, 0x62, 0x32, 0x2e, 0x30, 0x2d, 0x30, 0x20, 0x28,
+	0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x32, 0x34, 0x2e, 0x30, 0x29, 0xb7, 0x6c, 0x69, 0x62, 0x67, 0x74, 0x6b, 0x32, 0x2e, 0x30, 0x2d, 0x30,
+	0x20, 0x28, 0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x32, 0x34, 0x2e, 0x30, 0x29, 0xb3, 0x6c, 0x69, 0x62, 0x69, 0x63, 0x75, 0x34, 0x38, 0x20,
+	0x28, 0x3e, 0x3d, 0x20, 0x34, 0x2e, 0x38, 0x2d, 0x31, 0x29, 0xb9, 0x6c, 0x69, 0x62, 0x70, 0x61, 0x6e, 0x67, 0x6f, 0x31, 0x2e, 0x30,
+	0x2d, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x31, 0x34, 0x2e, 0x30, 0x29, 0xb6, 0x6c, 0x69, 0x62, 0x73, 0x73, 0x6c, 0x31,
+	0x2e, 0x30, 0x2e, 0x30, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x31, 0x2e, 0x30, 0x2e, 0x30, 0x29, 0xb3, 0x6c, 0x69, 0x62, 0x73, 0x74, 0x64,
+	0x63, 0x2b, 0x2b, 0x36, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x34, 0x2e, 0x36, 0x29, 0xa8, 0x6c, 0x69, 0x62, 0x78, 0x31, 0x31, 0x2d, 0x36,
+	0xb2, 0x6c, 0x69, 0x62, 0x78, 0x6d, 0x6c, 0x32, 0x20, 0x28, 0x3e, 0x3d, 0x20, 0x32, 0x2e, 0x37, 0x2e, 0x34, 0x29, 0xa8, 0x72, 0x64,
+	0x65, 0x73, 0x6b, 0x74, 0x6f, 0x70, 0xa5, 0x45, 0x78, 0x74, 0x72, 0x61, 0x88, 0xa3, 0x54, 0x61, 0x67, 0xbd, 0x72, 0x6f, 0x6c, 0x65,
+	0x3a, 0x3a, 0x70, 0x72, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x2c, 0x20, 0x75, 0x69, 0x74, 0x6f, 0x6f, 0x6c, 0x6b, 0x69, 0x74, 0x3a, 0x3a,
+	0x67, 0x74, 0x6b, 0xa8, 0x50, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, 0xa8, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0xaa,
+	0x4d, 0x61, 0x69, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0xda, 0x0, 0x28, 0x44, 0x65, 0x62, 0x69, 0x61, 0x6e, 0x20, 0x51, 0x41,
+	0x20, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x20, 0x3c, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x40, 0x71, 0x61, 0x2e, 0x64, 0x65,
+	0x62, 0x69, 0x61, 0x6e, 0x2e, 0x6f, 0x72, 0x67, 0x3e, 0xa8, 0x48, 0x6f, 0x6d, 0x65, 0x70, 0x61, 0x67, 0x65, 0xda, 0x0, 0x30, 0x68,
+	0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
+	0x70, 0x2f, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x76, 0x69, 0x65, 0x77, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x2d, 0x63, 0x6c, 0x69,
+	0x65, 0x6e, 0x74, 0xaf, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x6d, 0x64, 0x35, 0xda, 0x0, 0x20,
+	0x62, 0x34, 0x34, 0x64, 0x34, 0x39, 0x62, 0x34, 0x37, 0x61, 0x65, 0x30, 0x35, 0x35, 0x32, 0x63, 0x62, 0x66, 0x61, 0x64, 0x32, 0x31,
+	0x30, 0x64, 0x65, 0x32, 0x31, 0x63, 0x64, 0x65, 0x31, 0x39, 0xa7, 0x53, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0xab, 0x63, 0x6f, 0x6e,
+	0x74, 0x72, 0x69, 0x62, 0x2f, 0x78, 0x31, 0x31, 0xae, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x65, 0x64, 0x2d, 0x53, 0x69, 0x7a,
+	0x65, 0xa4, 0x31, 0x34, 0x35, 0x39, 0xab, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0xb9, 0x20, 0x56, 0x4d,
+	0x77, 0x61, 0x72, 0x65, 0x20, 0x56, 0x69, 0x65, 0x77, 0x20, 0x4f, 0x70, 0x65, 0x6e, 0x20, 0x43, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0xa,
+	0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x91, 0x82, 0xa9, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x73, 0x84, 0xa3, 0x4d, 0x44,
+	0x35, 0xda, 0x0, 0x20, 0x39, 0x63, 0x36, 0x31, 0x62, 0x35, 0x34, 0x65, 0x32, 0x36, 0x33, 0x38, 0x61, 0x31, 0x38, 0x66, 0x39, 0x35,
+	0x35, 0x61, 0x36, 0x39, 0x35, 0x62, 0x39, 0x31, 0x36, 0x32, 0x64, 0x36, 0x61, 0x66, 0xa4, 0x53, 0x48, 0x41, 0x31, 0xda, 0x0, 0x28,
+	0x35, 0x62, 0x37, 0x63, 0x39, 0x39, 0x65, 0x36, 0x34, 0x61, 0x37, 0x30, 0x66, 0x34, 0x66, 0x35, 0x30, 0x39, 0x62, 0x66, 0x61, 0x33,
+	0x61, 0x36, 0x37, 0x34, 0x30, 0x38, 0x38, 0x66, 0x66, 0x39, 0x63, 0x65, 0x66, 0x36, 0x38, 0x31, 0x36, 0x33, 0xa6, 0x53, 0x48, 0x41,
+	0x32, 0x35, 0x36, 0xda, 0x0, 0x40, 0x34, 0x61, 0x39, 0x65, 0x34, 0x62, 0x32, 0x64, 0x39, 0x62, 0x33, 0x64, 0x62, 0x31, 0x33, 0x66,
+	0x39, 0x61, 0x32, 0x39, 0x65, 0x35, 0x32, 0x32, 0x66, 0x33, 0x66, 0x66, 0x62, 0x62, 0x33, 0x34, 0x65, 0x65, 0x65, 0x39, 0x36, 0x66,
+	0x63, 0x36, 0x66, 0x33, 0x34, 0x61, 0x30, 0x36, 0x34, 0x37, 0x30, 0x34, 0x32, 0x61, 0x62, 0x31, 0x62, 0x35, 0x62, 0x30, 0x66, 0x32,
+	0x65, 0x30, 0x34, 0x64, 0xa4, 0x53, 0x69, 0x7a, 0x65, 0xce, 0x0, 0x7, 0xef, 0x90, 0xa8, 0x46, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d,
+	0x65, 0xda, 0x0, 0x5e, 0x70, 0x6f, 0x6f, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x69, 0x62, 0x2f, 0x76, 0x2f, 0x76, 0x6d, 0x77,
+	0x61, 0x72, 0x65, 0x2d, 0x76, 0x69, 0x65, 0x77, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x76,
+	0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x76, 0x69, 0x65, 0x77, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x2d, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74,
+	0x5f, 0x34, 0x2e, 0x35, 0x2e, 0x30, 0x2d, 0x32, 0x39, 0x37, 0x39, 0x37, 0x35, 0x2b, 0x64, 0x66, 0x73, 0x67, 0x2d, 0x34, 0x2b, 0x62,
+	0x31, 0x5f, 0x69, 0x33, 0x38, 0x36, 0x2e, 0x64, 0x65, 0x62, 0xa8, 0x49, 0x73, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0xc2, 0xa4, 0x4e,
+	0x61, 0x6d, 0x65, 0xb7, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x76, 0x69, 0x65, 0x77, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x2d, 0x63,
+	0x6c, 0x69, 0x65, 0x6e, 0x74, 0xaa, 0x50, 0x72, 0x65, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x73, 0xc0, 0xa8, 0x50, 0x72, 0x6f, 0x76,
+	0x69, 0x64, 0x65, 0x73, 0xc0, 0xaa, 0x52, 0x65, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x64, 0x73, 0xc0, 0xa6, 0x53, 0x6f, 0x75, 0x72,
+	0x63, 0x65, 0xda, 0x0, 0x2d, 0x76, 0x6d, 0x77, 0x61, 0x72, 0x65, 0x2d, 0x76, 0x69, 0x65, 0x77, 0x2d, 0x6f, 0x70, 0x65, 0x6e, 0x2d,
+	0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x20, 0x28, 0x34, 0x2e, 0x35, 0x2e, 0x30, 0x2d, 0x32, 0x39, 0x37, 0x39, 0x37, 0x35, 0x2b, 0x64,
+	0x66, 0x73, 0x67, 0x2d, 0x34, 0x29, 0xb2, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74,
+	0x75, 0x72, 0x65, 0xa0, 0xa8, 0x53, 0x75, 0x67, 0x67, 0x65, 0x73, 0x74, 0x73, 0xc0, 0xa7, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0xb6, 0x34, 0x2e, 0x35, 0x2e, 0x30, 0x2d, 0x32, 0x39, 0x37, 0x39, 0x37, 0x35, 0x2b, 0x64, 0x66, 0x73, 0x67, 0x2d, 0x34, 0x2b, 0x62, 0x31}

debian/package_deps.go

@@ -0,0 +1,30 @@
+package debian
+
+import (
+	"strings"
+)
+
+// PackageDependencies are various parsed dependencies
+type PackageDependencies struct {
+	Depends           []string
+	BuildDepends      []string
+	BuildDependsInDep []string
+	PreDepends        []string
+	Suggests          []string
+	Recommends        []string
+}
+
+func parseDependencies(input Stanza, key string) []string {
+	value, ok := input[key]
+	if !ok {
+		return nil
+	}
+
+	delete(input, key)
+
+	result := strings.Split(value, ",")
+	for i := range result {
+		result[i] = strings.TrimSpace(result[i])
+	}
+	return result
+}

debian/package_files.go

@@ -0,0 +1,78 @@
+package debian
+
+import (
+	"encoding/binary"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"hash/fnv"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// PackageFile is a single file entry in package
+type PackageFile struct {
+	// Filename is name of file for the package (without directory)
+	Filename string
+	// Hashes for the file
+	Checksums utils.ChecksumInfo
+	// Temporary field used while downloading, stored relative path on the mirror
+	downloadPath string
+}
+
+// Verify that package file is present and correct
+func (f *PackageFile) Verify(packagePool aptly.PackagePool) (bool, error) {
+	poolPath, err := packagePool.Path(f.Filename, f.Checksums.MD5)
+	if err != nil {
+		return false, err
+	}
+
+	st, err := os.Stat(poolPath)
+	if err != nil {
+		return false, nil
+	}
+
+	// verify size
+	// TODO: verify checksum if configured
+	return st.Size() == f.Checksums.Size, nil
+}
+
+// DownloadURL return relative URL to package download location
+func (f *PackageFile) DownloadURL() string {
+	return filepath.Join(f.downloadPath, f.Filename)
+}
+
+// PackageFiles is collection of package files
+type PackageFiles []PackageFile
+
+// Hash compute hash of all file items, sorting them first
+func (files PackageFiles) Hash() uint64 {
+	sort.Sort(files)
+
+	h := fnv.New64a()
+
+	for _, f := range files {
+		h.Write([]byte(f.Filename))
+		binary.Write(h, binary.BigEndian, f.Checksums.Size)
+		h.Write([]byte(f.Checksums.MD5))
+		h.Write([]byte(f.Checksums.SHA1))
+		h.Write([]byte(f.Checksums.SHA256))
+	}
+
+	return h.Sum64()
+}
+
+// Len returns number of files
+func (files PackageFiles) Len() int {
+	return len(files)
+}
+
+// Swap swaps elements
+func (files PackageFiles) Swap(i, j int) {
+	files[i], files[j] = files[j], files[i]
+}
+
+// Less compares by filename
+func (files PackageFiles) Less(i, j int) bool {
+	return files[i].Filename < files[j].Filename
+}

debian/package_files_test.go

@@ -0,0 +1,61 @@
+package debian
+
+import (
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+)
+
+type PackageFilesSuite struct {
+	files PackageFiles
+}
+
+var _ = Suite(&PackageFilesSuite{})
+
+func (s *PackageFilesSuite) SetUpTest(c *C) {
+	s.files = PackageFiles{PackageFile{
+		Filename:     "alien-arena-common_7.40-2_i386.deb",
+		downloadPath: "pool/contrib/a/alien-arena",
+		Checksums: utils.ChecksumInfo{
+			Size:   187518,
+			MD5:    "1e8cba92c41420aa7baa8a5718d67122",
+			SHA1:   "46955e48cad27410a83740a21d766ce362364024",
+			SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5",
+		}}}
+}
+
+func (s *PackageFilesSuite) TestVerify(c *C) {
+	packagePool := files.NewPackagePool(c.MkDir())
+	poolPath, _ := packagePool.Path(s.files[0].Filename, s.files[0].Checksums.MD5)
+
+	result, err := s.files[0].Verify(packagePool)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, false)
+
+	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	c.Assert(err, IsNil)
+
+	file, err := os.Create(poolPath)
+	c.Assert(err, IsNil)
+	file.WriteString("abcde")
+	file.Close()
+
+	result, err = s.files[0].Verify(packagePool)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, false)
+
+	s.files[0].Checksums.Size = 5
+	result, err = s.files[0].Verify(packagePool)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, true)
+}
+
+func (s *PackageFilesSuite) TestDownloadURL(c *C) {
+	c.Check(s.files[0].DownloadURL(), Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+}
+
+func (s *PackageFilesSuite) TestHash(c *C) {
+	c.Check(s.files.Hash(), Equals, uint64(0xc8901eedd79ac51b))
+}

debian/package_test.go

@@ -1,71 +1,76 @@
 package debian
 
 import (
-	"github.com/smira/aptly/database"
+	"bytes"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/utils"
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 )
 
-var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": "46955e48cad27410a83740a21d766ce362364024", "SHA256": "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
-
 type PackageSuite struct {
-	stanza Stanza
+	stanza       Stanza
+	sourceStanza Stanza
 }
 
 var _ = Suite(&PackageSuite{})
 
 func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
-}
 
-func (s *PackageSuite) TestPackageFileVerify(c *C) {
-	packageRepo := NewRepository(c.MkDir())
-	p := NewPackageFromControlFile(s.stanza)
-	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
-
-	result, err := p.Files[0].Verify(packageRepo)
-	c.Check(err, IsNil)
-	c.Check(result, Equals, false)
-
-	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
-	c.Assert(err, IsNil)
-
-	file, err := os.Create(poolPath)
-	c.Assert(err, IsNil)
-	file.WriteString("abcde")
-	file.Close()
-
-	result, err = p.Files[0].Verify(packageRepo)
-	c.Check(err, IsNil)
-	c.Check(result, Equals, false)
-
-	result, err = p.VerifyFiles(packageRepo)
-	c.Check(err, IsNil)
-	c.Check(result, Equals, false)
-
-	p.Files[0].Checksums.Size = 5
-	result, err = p.Files[0].Verify(packageRepo)
-	c.Check(err, IsNil)
-	c.Check(result, Equals, true)
-
-	result, err = p.VerifyFiles(packageRepo)
-	c.Check(err, IsNil)
-	c.Check(result, Equals, true)
+	buf := bytes.NewBufferString(sourcePackageMeta)
+	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza()
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
+	c.Check(p.IsSource, Equals, false)
 	c.Check(p.Name, Equals, "alien-arena-common")
 	c.Check(p.Version, Equals, "7.40-2")
 	c.Check(p.Architecture, Equals, "i386")
 	c.Check(p.Provides, DeepEquals, []string(nil))
-	c.Check(p.Files, HasLen, 1)
-	c.Check(p.Files[0].Filename, Equals, "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
-	c.Check(p.Files[0].Checksums.Size, Equals, int64(187518))
-	c.Check(p.Files[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122")
-	c.Check(p.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
+	c.Check(p.Files(), HasLen, 1)
+	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
+	c.Check(p.Files()[0].downloadPath, Equals, "pool/contrib/a/alien-arena")
+	c.Check(p.Files()[0].Checksums.Size, Equals, int64(187518))
+	c.Check(p.Files()[0].Checksums.MD5, Equals, "1e8cba92c41420aa7baa8a5718d67122")
+	c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
+}
+
+func (s *PackageSuite) TestNewSourceFromPara(c *C) {
+	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
+
+	c.Check(err, IsNil)
+	c.Check(p.IsSource, Equals, true)
+	c.Check(p.Name, Equals, "access-modifier-checker")
+	c.Check(p.Version, Equals, "1.0-4")
+	c.Check(p.Architecture, Equals, "source")
+	c.Check(p.SourceArchitecture, Equals, "all")
+	c.Check(p.Provides, IsNil)
+	c.Check(p.deps.BuildDepends, DeepEquals, []string{"cdbs", "debhelper (>= 7)", "default-jdk", "maven-debian-helper"})
+	c.Check(p.deps.BuildDependsInDep, DeepEquals, []string{"default-jdk-doc", "junit (>= 3.8.1)", "libannotation-indexer-java (>= 1.3)", "libannotation-indexer-java-doc", "libasm3-java", "libmaven-install-plugin-java", "libmaven-javadoc-plugin-java", "libmaven-scm-java", "libmaven2-core-java", "libmaven2-core-java-doc", "libmetainf-services-java", "libmetainf-services-java-doc", "libmaven-plugin-tools-java (>= 2.8)"})
+	c.Check(p.Files(), HasLen, 3)
+
+	c.Check(p.Files()[0].Filename, Equals, "access-modifier-checker_1.0-4.debian.tar.gz")
+	c.Check(p.Files()[0].downloadPath, Equals, "pool/main/a/access-modifier-checker")
+
+	c.Check(p.Files()[1].Filename, Equals, "access-modifier-checker_1.0-4.dsc")
+	c.Check(p.Files()[1].downloadPath, Equals, "pool/main/a/access-modifier-checker")
+	c.Check(p.Files()[1].Checksums.Size, Equals, int64(3))
+	c.Check(p.Files()[1].Checksums.MD5, Equals, "900150983cd24fb0d6963f7d28e17f72")
+	c.Check(p.Files()[1].Checksums.SHA1, Equals, "a9993e364706816aba3e25717850c26c9cd0d89d")
+	c.Check(p.Files()[1].Checksums.SHA256, Equals, "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+
+	c.Check(p.Files()[2].Filename, Equals, "access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(p.Files()[2].downloadPath, Equals, "pool/main/a/access-modifier-checker")
+	c.Check(p.Files()[2].Checksums.Size, Equals, int64(4))
+	c.Check(p.Files()[2].Checksums.MD5, Equals, "e2fc714c4727ee9395f324cd2e7f331f")
+	c.Check(p.Files()[2].Checksums.SHA1, Equals, "81fe8bfe87576c3ecb22426f8e57847382917acf")
+	c.Check(p.Files()[2].Checksums.SHA256, Equals, "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589")
+
+	c.Check(p.deps.Depends, IsNil)
 }
 
 func (s *PackageSuite) TestWithProvides(c *C) {
@@ -82,17 +87,8 @@ func (s *PackageSuite) TestWithProvides(c *C) {
 func (s *PackageSuite) TestKey(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
-	c.Check(p.Key(), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
-}
-
-func (s *PackageSuite) TestEncodeDecode(c *C) {
-	p := NewPackageFromControlFile(s.stanza)
-	encoded := p.Encode()
-	p2 := &Package{}
-	err := p2.Decode(encoded)
-
-	c.Assert(err, IsNil)
-	c.Assert(p2, DeepEquals, p)
+	c.Check(p.Key(""), DeepEquals, []byte("Pi386 alien-arena-common 7.40-2"))
+	c.Check(p.Key("xD"), DeepEquals, []byte("xDPi386 alien-arena-common 7.40-2"))
 }
 
 func (s *PackageSuite) TestStanza(c *C) {
@@ -100,11 +96,16 @@ func (s *PackageSuite) TestStanza(c *C) {
 	stanza := p.Stanza()
 
 	c.Assert(stanza, DeepEquals, s.stanza)
+
+	p, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+	stanza = p.Stanza()
+
+	c.Assert(stanza, DeepEquals, s.sourceStanza)
 }
 
 func (s *PackageSuite) TestString(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
-	c.Assert(p.String(), Equals, "alien-arena-common-7.40-2_i386")
+	c.Assert(p.String(), Equals, "alien-arena-common_7.40-2_i386")
 }
 
 func (s *PackageSuite) TestEquals(c *C) {
@@ -113,12 +114,30 @@ func (s *PackageSuite) TestEquals(c *C) {
 	p2 := NewPackageFromControlFile(packageStanza.Copy())
 	c.Check(p.Equals(p2), Equals, true)
 
-	p2.Depends = []string{"package1"}
-	c.Check(p.Equals(p2), Equals, false)
+	p2.deps.Depends = []string{"package1"}
+	c.Check(p.Equals(p2), Equals, true) // strange, but Equals doesn't check deep
 
 	p2 = NewPackageFromControlFile(packageStanza.Copy())
-	p2.Files[0].Checksums.MD5 = "abcdefabcdef"
+	files := p2.Files()
+	files[0].Checksums.MD5 = "abcdefabcdef"
+	p2.UpdateFiles(files)
 	c.Check(p.Equals(p2), Equals, false)
+
+	so, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+	so2, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+
+	c.Check(so.Equals(so2), Equals, true)
+
+	files = so2.Files()
+	files[2].Checksums.MD5 = "abcde"
+	so2.UpdateFiles(files)
+	c.Check(so.Equals(so2), Equals, false)
+
+	so2, _ = NewSourcePackageFromControlFile(s.sourceStanza.Copy())
+	files = so2.Files()
+	files[1].Filename = "other.deb"
+	so2.UpdateFiles(files)
+	c.Check(so.Equals(so2), Equals, false)
 }
 
 func (s *PackageSuite) TestMatchesArchitecture(c *C) {
@@ -131,6 +150,49 @@ func (s *PackageSuite) TestMatchesArchitecture(c *C) {
 	p = NewPackageFromControlFile(s.stanza)
 	c.Check(p.MatchesArchitecture("i386"), Equals, true)
 	c.Check(p.MatchesArchitecture("amd64"), Equals, true)
+	c.Check(p.MatchesArchitecture("source"), Equals, false)
+
+	p, _ = NewSourcePackageFromControlFile(s.sourceStanza)
+	c.Check(p.MatchesArchitecture("source"), Equals, true)
+	c.Check(p.MatchesArchitecture("amd64"), Equals, false)
+}
+
+func (s *PackageSuite) TestMatchesDependency(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
+
+	// exact match
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
+
+	// different name
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena", Architecture: "i386", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
+
+	// different version
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionEqual, Version: "7.40-3"}), Equals, false)
+
+	// different arch
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "amd64", Relation: VersionEqual, Version: "7.40-2"}), Equals, false)
+
+	// empty arch
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "", Relation: VersionEqual, Version: "7.40-2"}), Equals, true)
+
+	// version don't care
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionDontCare, Version: ""}), Equals, true)
+
+	// >
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionGreater, Version: "7.40-2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionGreater, Version: "7.40-1"}), Equals, true)
+
+	// <
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLess, Version: "7.40-2"}), Equals, false)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLess, Version: "7.40-3"}), Equals, true)
+
+	// >=
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionGreaterOrEqual, Version: "7.40-2"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionGreaterOrEqual, Version: "7.40-3"}), Equals, false)
+
+	// <=
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-2"}), Equals, true)
+	c.Check(p.MatchesDependency(Dependency{Pkg: "alien-arena-common", Architecture: "i386", Relation: VersionLessOrEqual, Version: "7.40-1"}), Equals, false)
 }
 
 func (s *PackageSuite) TestGetDependencies(c *C) {
@@ -138,6 +200,16 @@ func (s *PackageSuite) TestGetDependencies(c *C) {
 	c.Check(p.GetDependencies(0), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)"})
 	c.Check(p.GetDependencies(DepFollowSuggests), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)", "alien-arena-mars"})
 	c.Check(p.GetDependencies(DepFollowSuggests|DepFollowRecommends), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)", "aliean-arena-luna", "alien-arena-mars"})
+
+	c.Check(p.GetDependencies(DepFollowSource), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)", "alien-arena (= 7.40-2) {source}"})
+	p.Source = "alien-arena (7.40-3)"
+	c.Check(p.GetDependencies(DepFollowSource), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)", "alien-arena (7.40-3) {source}"})
+	p.Source = ""
+	c.Check(p.GetDependencies(DepFollowSource), DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)", "dpkg (>= 1.6)", "alien-arena-common (= 7.40-2) {source}"})
+
+	p, _ = NewSourcePackageFromControlFile(s.sourceStanza)
+	c.Check(p.GetDependencies(0), DeepEquals, []string{})
+	c.Check(p.GetDependencies(DepFollowBuild), DeepEquals, []string{"cdbs", "debhelper (>= 7)", "default-jdk", "maven-debian-helper", "default-jdk-doc", "junit (>= 3.8.1)", "libannotation-indexer-java (>= 1.3)", "libannotation-indexer-java-doc", "libasm3-java", "libmaven-install-plugin-java", "libmaven-javadoc-plugin-java", "libmaven-scm-java", "libmaven2-core-java", "libmaven2-core-java-doc", "libmetainf-services-java", "libmetainf-services-java-doc", "libmaven-plugin-tools-java (>= 2.8)"})
 }
 
 func (s *PackageSuite) TestPoolDirectory(c *C) {
@@ -165,10 +237,11 @@ func (s *PackageSuite) TestPoolDirectory(c *C) {
 }
 
 func (s *PackageSuite) TestLinkFromPool(c *C) {
-	packageRepo := NewRepository(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir())
+	publishedStorage := files.NewPublishedStorage(c.MkDir())
 	p := NewPackageFromControlFile(s.stanza)
 
-	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
+	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
 	c.Assert(err, IsNil)
 
@@ -176,24 +249,42 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 	c.Assert(err, IsNil)
 	file.Close()
 
-	err = p.LinkFromPool(packageRepo, "", "non-free")
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
 	c.Check(err, IsNil)
-	c.Check(p.Files[0].Filename, Equals, "pool/non-free/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
+	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
+	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
+
+	p.IsSource = true
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free")
+	c.Check(err, IsNil)
+	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
+}
+
+func (s *PackageSuite) TestFilepathList(c *C) {
+	packagePool := files.NewPackagePool(c.MkDir())
+	p := NewPackageFromControlFile(s.stanza)
+
+	list, err := p.FilepathList(packagePool)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"1e/8c/alien-arena-common_7.40-2_i386.deb"})
 }
 
 func (s *PackageSuite) TestDownloadList(c *C) {
-	packageRepo := NewRepository(c.MkDir())
+	packagePool := files.NewPackagePool(c.MkDir())
 	p := NewPackageFromControlFile(s.stanza)
-	p.Files[0].Checksums.Size = 5
-	poolPath, _ := packageRepo.PoolPath(p.Files[0].Filename, p.Files[0].Checksums.MD5)
+	p.Files()[0].Checksums.Size = 5
+	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 
-	list, err := p.DownloadList(packageRepo)
+	list, err := p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{
 		PackageDownloadTask{
 			RepoURI:         "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			DestinationPath: poolPath,
-			Size:            5}})
+			Checksums: utils.ChecksumInfo{Size: 5,
+				MD5:    "1e8cba92c41420aa7baa8a5718d67122",
+				SHA1:   "46955e48cad27410a83740a21d766ce362364024",
+				SHA256: "eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5"}}})
 
 	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
 	c.Assert(err, IsNil)
@@ -203,34 +294,68 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 	file.WriteString("abcde")
 	file.Close()
 
-	list, err = p.DownloadList(packageRepo)
+	list, err = p.DownloadList(packagePool)
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []PackageDownloadTask{})
 }
 
-type PackageCollectionSuite struct {
-	collection *PackageCollection
-	p          *Package
-	db         database.Storage
-}
+func (s *PackageSuite) TestVerifyFiles(c *C) {
+	p := NewPackageFromControlFile(s.stanza)
 
-var _ = Suite(&PackageCollectionSuite{})
+	packagePool := files.NewPackagePool(c.MkDir())
+	poolPath, _ := packagePool.Path(p.Files()[0].Filename, p.Files()[0].Checksums.MD5)
 
-func (s *PackageCollectionSuite) SetUpTest(c *C) {
-	s.p = NewPackageFromControlFile(packageStanza.Copy())
-	s.db, _ = database.OpenDB(c.MkDir())
-	s.collection = NewPackageCollection(s.db)
-}
-
-func (s *PackageCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *PackageCollectionSuite) TestUpdateByKey(c *C) {
-	err := s.collection.Update(s.p)
+	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
 	c.Assert(err, IsNil)
 
-	p2, err := s.collection.ByKey(s.p.Key())
+	file, err := os.Create(poolPath)
 	c.Assert(err, IsNil)
-	c.Assert(p2.Equals(s.p), Equals, true)
+	file.WriteString("abcde")
+	file.Close()
+
+	result, err := p.VerifyFiles(packagePool)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, false)
+
+	p.Files()[0].Checksums.Size = 5
+
+	result, err = p.VerifyFiles(packagePool)
+	c.Check(err, IsNil)
+	c.Check(result, Equals, true)
 }
+
+var packageStanza = Stanza{"Source": "alien-arena", "Pre-Depends": "dpkg (>= 1.6)", "Suggests": "alien-arena-mars", "Recommends": "aliean-arena-luna", "Depends": "libc6 (>= 2.7), alien-arena-data (>= 7.40)", "Filename": "pool/contrib/a/alien-arena/alien-arena-common_7.40-2_i386.deb", "SHA1": " 46955e48cad27410a83740a21d766ce362364024", "SHA256": " eb4afb9885cba6dc70cccd05b910b2dbccc02c5900578be5e99f0d3dbf9d76a5", "Priority": "extra", "Maintainer": "Debian Games Team <pkg-games-devel@lists.alioth.debian.org>", "Description": "Common files for Alien Arena client and server ALIEN ARENA is a standalone 3D first person online deathmatch shooter\n crafted from the original source code of Quake II and Quake III, released\n by id Software under the GPL license. With features including 32 bit\n graphics, new particle engine and effects, light blooms, reflective water,\n hi resolution textures and skins, hi poly models, stain maps, ALIEN ARENA\n pushes the envelope of graphical beauty rivaling today's top games.\n .\n This package installs the common files for Alien Arena.\n", "Homepage": "http://red.planetarena.org", "Tag": "role::app-data, role::shared-lib, special::auto-inst-parts", "Installed-Size": "456", "Version": "7.40-2", "Replaces": "alien-arena (<< 7.33-1)", "Size": "187518", "MD5sum": "1e8cba92c41420aa7baa8a5718d67122", "Package": "alien-arena-common", "Section": "contrib/games", "Architecture": "i386"}
+
+const sourcePackageMeta = `Package: access-modifier-checker
+Binary: libaccess-modifier-checker-java, libaccess-modifier-checker-java-doc
+Version: 1.0-4
+Maintainer: Debian Java Maintainers <pkg-java-maintainers@lists.alioth.debian.org>
+Uploaders: James Page <james.page@ubuntu.com>
+Build-Depends: cdbs, debhelper (>= 7), default-jdk, maven-debian-helper
+Build-Depends-Indep: default-jdk-doc, junit (>= 3.8.1), libannotation-indexer-java (>= 1.3), libannotation-indexer-java-doc, libasm3-java, libmaven-install-plugin-java, libmaven-javadoc-plugin-java, libmaven-scm-java, libmaven2-core-java, libmaven2-core-java-doc, libmetainf-services-java, libmetainf-services-java-doc, libmaven-plugin-tools-java (>= 2.8)
+Architecture: all
+Standards-Version: 3.9.3
+Format: 3.0 (quilt)
+Files:
+ ab56b4d92b40713acc5af89985d4b786 5 access-modifier-checker_1.0-4.debian.tar.gz
+ 900150983cd24fb0d6963f7d28e17f72 3 access-modifier-checker_1.0-4.dsc
+ e2fc714c4727ee9395f324cd2e7f331f 4 access-modifier-checker_1.0.orig.tar.gz
+Dm-Upload-Allowed: yes
+Vcs-Browser: http://git.debian.org/?p=pkg-java/access-modifier-checker.git
+Vcs-Git: git://git.debian.org/git/pkg-java/access-modifier-checker.git
+Checksums-Sha1:
+ 03de6c570bfe24bfc328ccd7ca46b76eadaf4334 5 access-modifier-checker_1.0-4.debian.tar.gz
+ a9993e364706816aba3e25717850c26c9cd0d89d 3 access-modifier-checker_1.0-4.dsc
+ 81fe8bfe87576c3ecb22426f8e57847382917acf 4 access-modifier-checker_1.0.orig.tar.gz
+Checksums-Sha256:
+ 36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c 5 access-modifier-checker_1.0-4.debian.tar.gz
+ ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad 3 access-modifier-checker_1.0-4.dsc
+ 88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589 4 access-modifier-checker_1.0.orig.tar.gz
+Homepage: https://github.com/kohsuke/access-modifier
+Package-List:
+ libaccess-modifier-checker-java deb java optional
+ libaccess-modifier-checker-java-doc deb doc optional
+Directory: pool/main/a/access-modifier-checker
+Priority: source
+Section: java
+`

debian/ppa.go

@@ -0,0 +1,54 @@
+package debian
+
+import (
+	"fmt"
+	"github.com/smira/aptly/utils"
+	"os/exec"
+	"regexp"
+	"strings"
+)
+
+var ppaRegexp = regexp.MustCompile("^ppa:([^/]+)/(.+)$")
+
+// ParsePPA converts ppa URL like ppa:user/ppa-name to full HTTP url
+func ParsePPA(ppaURL string) (url string, distribution string, components []string, err error) {
+	matches := ppaRegexp.FindStringSubmatch(ppaURL)
+	if matches == nil {
+		err = fmt.Errorf("unable to parse ppa URL: %v", ppaURL)
+		return
+	}
+
+	distributorID := utils.Config.PpaDistributorID
+	if distributorID == "" {
+		distributorID, err = getDistributorID()
+		if err != nil {
+			err = fmt.Errorf("unable to figure out Distributor ID: %s, please set config option ppaDistributorID", err)
+			return
+		}
+	}
+
+	codename := utils.Config.PpaCodename
+	if codename == "" {
+		codename, err = getCodename()
+		if err != nil {
+			err = fmt.Errorf("unable to figure out Codename: %s, please set config option ppaCodename", err)
+			return
+		}
+	}
+
+	distribution = codename
+	components = []string{"main"}
+	url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID)
+
+	return
+}
+
+func getCodename() (string, error) {
+	out, err := exec.Command("lsb_release", "-sc").Output()
+	return strings.TrimSpace(string(out)), err
+}
+
+func getDistributorID() (string, error) {
+	out, err := exec.Command("lsb_release", "-si").Output()
+	return strings.ToLower(strings.TrimSpace(string(out))), err
+}

debian/ppa_test.go

@@ -0,0 +1,34 @@
+package debian
+
+import (
+	"github.com/smira/aptly/utils"
+	. "launchpad.net/gocheck"
+)
+
+type PpaSuite struct {
+	savedConfig utils.ConfigStructure
+}
+
+var _ = Suite(&PpaSuite{})
+
+func (s *PpaSuite) SetUpTest(c *C) {
+	s.savedConfig = utils.Config
+}
+
+func (s *PpaSuite) TearDownTest(c *C) {
+	utils.Config = s.savedConfig
+}
+
+func (s *PpaSuite) TestParsePPA(c *C) {
+	_, _, _, err := ParsePPA("ppa:dedeed")
+	c.Check(err, ErrorMatches, "unable to parse ppa URL.*")
+
+	utils.Config.PpaDistributorID = "debian"
+	utils.Config.PpaCodename = "wheezy"
+
+	url, distribution, components, err := ParsePPA("ppa:user/project")
+	c.Check(err, IsNil)
+	c.Check(url, Equals, "http://ppa.launchpad.net/user/project/debian")
+	c.Check(distribution, Equals, "wheezy")
+	c.Check(components, DeepEquals, []string{"main"})
+}

debian/publish.go

@@ -5,11 +5,13 @@ import (
 	"bytes"
 	"code.google.com/p/go-uuid/uuid"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 )
@@ -91,46 +93,65 @@ func (p *PublishedRepo) Decode(input []byte) error {
 }
 
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
-func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageCollection, signer utils.Signer) error {
-	err := repo.MkDir(filepath.Join(p.Prefix, "pool"))
+func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorage aptly.PublishedStorage, packageCollection *PackageCollection, signer utils.Signer, progress aptly.Progress) error {
+	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
 	if err != nil {
 		return err
 	}
 	basePath := filepath.Join(p.Prefix, "dists", p.Distribution)
-	err = repo.MkDir(basePath)
+	err = publishedStorage.MkDir(basePath)
 	if err != nil {
 		return err
 	}
 
+	if progress != nil {
+		progress.Printf("Loading packages...\n")
+	}
+
 	// Load all packages
-	list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection)
+	list, err := NewPackageListFromRefList(p.snapshot.RefList(), packageCollection, progress)
 	if err != nil {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
 	if list.Len() == 0 {
-		return fmt.Errorf("repository is empty, can't publish")
+		return fmt.Errorf("snapshot is empty")
 	}
 
 	if len(p.Architectures) == 0 {
-		p.Architectures = list.Architectures()
+		p.Architectures = list.Architectures(true)
 	}
 
 	if len(p.Architectures) == 0 {
 		return fmt.Errorf("unable to figure out list of architectures, please supply explicit list")
 	}
 
-	generatedFiles := map[string]*utils.ChecksumInfo{}
+	sort.Strings(p.Architectures)
+
+	generatedFiles := map[string]utils.ChecksumInfo{}
+
+	if progress != nil {
+		progress.Printf("Generating metadata files and linking package files...\n")
+	}
 
 	// For all architectures, generate release file
 	for _, arch := range p.Architectures {
-		relativePath := filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
-		err = repo.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
+		if progress != nil {
+			progress.InitBar(int64(list.Len()), false)
+		}
+
+		var relativePath string
+		if arch == "source" {
+			relativePath = filepath.Join(p.Component, "source", "Sources")
+		} else {
+			relativePath = filepath.Join(p.Component, fmt.Sprintf("binary-%s", arch), "Packages")
+		}
+		err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
 		if err != nil {
 			return err
 		}
 
-		packagesFile, err := repo.CreateFile(filepath.Join(basePath, relativePath))
+		packagesFile, err := publishedStorage.CreateFile(filepath.Join(basePath, relativePath))
 		if err != nil {
 			return fmt.Errorf("unable to creates Packages file: %s", err)
 		}
@@ -138,8 +159,11 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 		bufWriter := bufio.NewWriter(packagesFile)
 
 		err = list.ForEach(func(pkg *Package) error {
+			if progress != nil {
+				progress.AddBar(1)
+			}
 			if pkg.MatchesArchitecture(arch) {
-				err = pkg.LinkFromPool(repo, p.Prefix, p.Component)
+				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Component)
 				if err != nil {
 					return err
 				}
@@ -153,13 +177,17 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 					return err
 				}
 
+				pkg.files = nil
+				pkg.deps = nil
+				pkg.extra = nil
+
 			}
 
 			return nil
 		})
 
 		if err != nil {
-			return fmt.Errorf("unable to creates process packages: %s", err)
+			return fmt.Errorf("unable to process packages: %s", err)
 		}
 
 		err = bufWriter.Flush()
@@ -174,24 +202,27 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 
 		packagesFile.Close()
 
-		checksumInfo, err := repo.ChecksumsForFile(filepath.Join(basePath, relativePath))
+		checksumInfo, err := publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath))
 		if err != nil {
 			return fmt.Errorf("unable to collect checksums: %s", err)
 		}
 		generatedFiles[relativePath] = checksumInfo
 
-		checksumInfo, err = repo.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
+		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".gz"))
 		if err != nil {
 			return fmt.Errorf("unable to collect checksums: %s", err)
 		}
 		generatedFiles[relativePath+".gz"] = checksumInfo
 
-		checksumInfo, err = repo.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
+		checksumInfo, err = publishedStorage.ChecksumsForFile(filepath.Join(basePath, relativePath+".bz2"))
 		if err != nil {
 			return fmt.Errorf("unable to collect checksums: %s", err)
 		}
 		generatedFiles[relativePath+".bz2"] = checksumInfo
 
+		if progress != nil {
+			progress.ShutdownBar()
+		}
 	}
 
 	release := make(Stanza)
@@ -200,8 +231,8 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 	release["Codename"] = p.Distribution
 	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
 	release["Components"] = p.Component
-	release["Architectures"] = strings.Join(p.Architectures, " ")
-	release["Description"] = "Generated by aptly\n"
+	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{"source"}), " ")
+	release["Description"] = " Generated by aptly\n"
 	release["MD5Sum"] = "\n"
 	release["SHA1"] = "\n"
 	release["SHA256"] = "\n"
@@ -212,7 +243,7 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
 	}
 
-	releaseFile, err := repo.CreateFile(filepath.Join(basePath, "Release"))
+	releaseFile, err := publishedStorage.CreateFile(filepath.Join(basePath, "Release"))
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -232,14 +263,21 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 	releaseFilename := releaseFile.Name()
 	releaseFile.Close()
 
-	err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
-	if err != nil {
-		return fmt.Errorf("unable to sign Release file: %s", err)
+	// Signing files might output to console, so flush progress writer first
+	if progress != nil {
+		progress.Flush()
 	}
 
-	err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"))
-	if err != nil {
-		return fmt.Errorf("unable to sign Release file: %s", err)
+	if signer != nil {
+		err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to sign Release file: %s", err)
+		}
+
+		err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"))
+		if err != nil {
+			return fmt.Errorf("unable to sign Release file: %s", err)
+		}
 	}
 
 	return nil
@@ -248,23 +286,23 @@ func (p *PublishedRepo) Publish(repo *Repository, packageCollection *PackageColl
 // RemoveFiles removes files that were created by Publish
 //
 // It can remove prefix fully, and part of pool (for specific component)
-func (p *PublishedRepo) RemoveFiles(repo *Repository, removePrefix, removePoolComponent bool) error {
+func (p *PublishedRepo) RemoveFiles(publishedStorage aptly.PublishedStorage, removePrefix, removePoolComponent bool) error {
 	if removePrefix {
-		err := repo.RemoveDirs(filepath.Join(p.Prefix, "dists"))
+		err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists"))
 		if err != nil {
 			return err
 		}
 
-		return repo.RemoveDirs(filepath.Join(p.Prefix, "pool"))
+		return publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool"))
 	}
 
-	err := repo.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
+	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution))
 	if err != nil {
 		return err
 	}
 
 	if removePoolComponent {
-		err = repo.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
+		err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "pool", p.Component))
 		if err != nil {
 			return err
 		}
@@ -365,6 +403,17 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 	return nil, fmt.Errorf("published repo with uuid %s not found", uuid)
 }
 
+// BySnapshot looks up repository by snapshot source
+func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
+	result := make([]*PublishedRepo, 0)
+	for _, r := range collection.list {
+		if r.SnapshotUUID == snapshot.UUID {
+			result = append(result, r)
+		}
+	}
+	return result
+}
+
 // ForEach runs method for each repository
 func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
 	var err error
@@ -383,7 +432,7 @@ func (collection *PublishedRepoCollection) Len() int {
 }
 
 // Remove removes published repository, cleaning up directories, files
-func (collection *PublishedRepoCollection) Remove(packageRepo *Repository, prefix, distribution string) error {
+func (collection *PublishedRepoCollection) Remove(publishedStorage aptly.PublishedStorage, prefix, distribution string) error {
 	repo, err := collection.ByPrefixDistribution(prefix, distribution)
 	if err != nil {
 		return err
@@ -406,7 +455,7 @@ func (collection *PublishedRepoCollection) Remove(packageRepo *Repository, prefi
 		}
 	}
 
-	err = repo.RemoveFiles(packageRepo, removePrefix, removePoolComponent)
+	err = repo.RemoveFiles(publishedStorage, removePrefix, removePoolComponent)
 	if err != nil {
 		return err
 	}

debian/publish_test.go

@@ -2,16 +2,37 @@ package debian
 
 import (
 	"errors"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/files"
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 )
 
+type pathExistsChecker struct {
+	*CheckerInfo
+}
+
+var PathExists = &pathExistsChecker{
+	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
+}
+
+func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
+	_, err := os.Stat(params[0].(string))
+	return err == nil, ""
+}
+
 type NullSigner struct{}
 
-func (n *NullSigner) SetKey(keyRef string) {
+func (n *NullSigner) Init() error {
+	return nil
+}
 
+func (n *NullSigner) SetKey(keyRef string) {
+}
+
+func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 
 func (n *NullSigner) DetachedSign(source string, destination string) error {
@@ -25,7 +46,9 @@ func (n *NullSigner) ClearSign(source string, destination string) error {
 type PublishedRepoSuite struct {
 	PackageListMixinSuite
 	repo              *PublishedRepo
-	packageRepo       *Repository
+	root              string
+	publishedStorage  aptly.PublishedStorage
+	packagePool       aptly.PackagePool
 	snapshot          *Snapshot
 	db                database.Storage
 	packageCollection *PackageCollection
@@ -38,9 +61,11 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.db, _ = database.OpenDB(c.MkDir())
 
-	s.packageRepo = NewRepository(c.MkDir())
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.packagePool = files.NewPackagePool(s.root)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	repo.packageRefs = s.reflist
 
 	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
@@ -52,7 +77,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.packageCollection.Update(s.p2)
 	s.packageCollection.Update(s.p3)
 
-	poolPath, _ := s.packageRepo.PoolPath(s.p1.Files[0].Filename, s.p1.Files[0].Checksums.MD5)
+	poolPath, _ := s.packagePool.Path(s.p1.Files()[0].Filename, s.p1.Files()[0].Checksums.MD5)
 	err := os.MkdirAll(filepath.Dir(poolPath), 0755)
 	f, err := os.Create(poolPath)
 	c.Assert(err, IsNil)
@@ -129,12 +154,12 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
 }
 
 func (s *PublishedRepoSuite) TestPublish(c *C) {
-	err := s.repo.Publish(s.packageRepo, s.packageCollection, &NullSigner{})
+	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, &NullSigner{}, nil)
 	c.Assert(err, IsNil)
 
 	c.Check(s.repo.Architectures, DeepEquals, []string{"i386"})
 
-	rf, err := os.Open(filepath.Join(s.packageRepo.RootPath, "public/ppa/dists/squeeze/Release"))
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 
 	cfr := NewControlFileReader(rf)
@@ -145,7 +170,7 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Check(st["Components"], Equals, "main")
 	c.Check(st["Architectures"], Equals, "i386")
 
-	pf, err := os.Open(filepath.Join(s.packageRepo.RootPath, "public/ppa/dists/squeeze/main/binary-i386/Packages"))
+	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
 	c.Assert(err, IsNil)
 
 	cfr = NewControlFileReader(pf)
@@ -161,10 +186,17 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
-	_, err = os.Stat(filepath.Join(s.packageRepo.RootPath, "public/ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
+	_, err = os.Stat(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
+	err := s.repo.Publish(s.packagePool, s.publishedStorage, s.packageCollection, nil, nil)
+	c.Assert(err, IsNil)
+
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"), PathExists)
+}
+
 func (s *PublishedRepoSuite) TestString(c *C) {
 	c.Check(s.repo.String(), Equals,
 		"ppa/squeeze (main) publishes [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze")
@@ -223,7 +255,7 @@ func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
 	s.db.Close()
 }
 
-func (s *PublishedRepoCollectionSuite) TestAddByName(c *C) {
+func (s *PublishedRepoCollectionSuite) TestAddByPrefixDistribution(c *C) {
 	r, err := s.collection.ByPrefixDistribution("ppa", "anaconda")
 	c.Assert(err, ErrorMatches, "*.not found")
 
@@ -296,17 +328,12 @@ func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
 	c.Assert(err, Equals, e)
 }
 
-type pathExistsChecker struct {
-	*CheckerInfo
-}
+func (s *PublishedRepoCollectionSuite) TestBySnapshot(c *C) {
+	c.Check(s.collection.Add(s.repo1), IsNil)
+	c.Check(s.collection.Add(s.repo2), IsNil)
 
-var PathExists = &pathExistsChecker{
-	&CheckerInfo{Name: "PathExists", Params: []string{"path"}},
-}
-
-func (checker *pathExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
-	_, err := os.Stat(params[0].(string))
-	return err == nil, ""
+	c.Check(s.collection.BySnapshot(s.snap1), DeepEquals, []*PublishedRepo{s.repo1})
+	c.Check(s.collection.BySnapshot(s.snap2), DeepEquals, []*PublishedRepo{s.repo2})
 }
 
 type PublishedRepoRemoveSuite struct {
@@ -314,7 +341,8 @@ type PublishedRepoRemoveSuite struct {
 	db                         database.Storage
 	snapshotCollection         *SnapshotCollection
 	collection                 *PublishedRepoCollection
-	packageRepo                *Repository
+	root                       string
+	publishedStorage           aptly.PublishedStorage
 	snap1                      *Snapshot
 	repo1, repo2, repo3, repo4 *PublishedRepo
 }
@@ -341,14 +369,15 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.collection.Add(s.repo3)
 	s.collection.Add(s.repo4)
 
-	s.packageRepo = NewRepository(c.MkDir())
-	s.packageRepo.MkDir("ppa/dists/anaconda")
-	s.packageRepo.MkDir("ppa/dists/meduza")
-	s.packageRepo.MkDir("ppa/dists/osminog")
-	s.packageRepo.MkDir("ppa/pool/main")
-	s.packageRepo.MkDir("ppa/pool/contrib")
-	s.packageRepo.MkDir("dists/anaconda")
-	s.packageRepo.MkDir("pool/main")
+	s.root = c.MkDir()
+	s.publishedStorage = files.NewPublishedStorage(s.root)
+	s.publishedStorage.MkDir("ppa/dists/anaconda")
+	s.publishedStorage.MkDir("ppa/dists/meduza")
+	s.publishedStorage.MkDir("ppa/dists/osminog")
+	s.publishedStorage.MkDir("ppa/pool/main")
+	s.publishedStorage.MkDir("ppa/pool/contrib")
+	s.publishedStorage.MkDir("dists/anaconda")
+	s.publishedStorage.MkDir("pool/main")
 }
 
 func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
@@ -356,54 +385,54 @@ func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.packageRepo, false, false)
+	s.repo1.RemoveFiles(s.publishedStorage, false, false)
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.packageRepo, false, true)
+	s.repo1.RemoveFiles(s.publishedStorage, false, true)
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.packageRepo, true, true)
+	s.repo1.RemoveFiles(s.publishedStorage, true, true)
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.packageRepo, true, true)
+	s.repo2.RemoveFiles(s.publishedStorage, true, true)
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), Not(PathExists))
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
-	err := s.collection.Remove(s.packageRepo, "ppa", "anaconda")
+	err := s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByPrefixDistribution("ppa", "anaconda")
@@ -413,31 +442,31 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo1and2(c *C) {
 	_, err = collection.ByPrefixDistribution("ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 
-	err = s.collection.Remove(s.packageRepo, "ppa", "anaconda")
+	err = s.collection.Remove(s.publishedStorage, "ppa", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
-	err = s.collection.Remove(s.packageRepo, "ppa", "meduza")
+	err = s.collection.Remove(s.publishedStorage, "ppa", "meduza")
 	c.Check(err, IsNil)
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/main"), PathExists)
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
-	err := s.collection.Remove(s.packageRepo, ".", "anaconda")
+	err := s.collection.Remove(s.publishedStorage, ".", "anaconda")
 	c.Check(err, IsNil)
 
 	_, err = s.collection.ByPrefixDistribution(".", "anaconda")
@@ -447,11 +476,11 @@ func (s *PublishedRepoRemoveSuite) TestRemoveRepo3(c *C) {
 	_, err = collection.ByPrefixDistribution(".", "anaconda")
 	c.Check(err, ErrorMatches, ".*not found")
 
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/anaconda"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/meduza"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/dists/osminog"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/main"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "ppa/pool/contrib"), PathExists)
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "dists/"), Not(PathExists))
-	c.Check(filepath.Join(s.packageRepo.PublicPath(), "pool/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/osminog"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/main"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/pool/contrib"), PathExists)
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "dists/"), Not(PathExists))
+	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "pool/"), Not(PathExists))
 }

debian/reflist.go

@@ -0,0 +1,286 @@
+package debian
+
+import (
+	"bytes"
+	"github.com/ugorji/go/codec"
+	"sort"
+)
+
+// PackageRefList is a list of keys of packages, this is basis for snapshot
+// and similar stuff
+//
+// Refs are sorted in lexicographical order
+type PackageRefList struct {
+	// List of package keys
+	Refs [][]byte
+}
+
+// Verify interface
+var (
+	_ sort.Interface = &PackageRefList{}
+)
+
+// NewPackageRefList creates empty PackageRefList
+func NewPackageRefList() *PackageRefList {
+	return &PackageRefList{}
+}
+
+// NewPackageRefListFromPackageList creates PackageRefList from PackageList
+func NewPackageRefListFromPackageList(list *PackageList) *PackageRefList {
+	reflist := &PackageRefList{}
+	reflist.Refs = make([][]byte, list.Len())
+
+	i := 0
+	for _, p := range list.packages {
+		reflist.Refs[i] = p.Key("")
+		i++
+	}
+
+	sort.Sort(reflist)
+
+	return reflist
+}
+
+// Len returns number of refs
+func (l *PackageRefList) Len() int {
+	return len(l.Refs)
+}
+
+// Swap swaps two refs
+func (l *PackageRefList) Swap(i, j int) {
+	l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i]
+}
+
+// Compare compares two refs in lexographical order
+func (l *PackageRefList) Less(i, j int) bool {
+	return bytes.Compare(l.Refs[i], l.Refs[j]) < 0
+}
+
+// Encode does msgpack encoding of PackageRefList
+func (l *PackageRefList) Encode() []byte {
+	var buf bytes.Buffer
+
+	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
+	encoder.Encode(l)
+
+	return buf.Bytes()
+}
+
+// Decode decodes msgpack representation into PackageRefLit
+func (l *PackageRefList) Decode(input []byte) error {
+	decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
+	return decoder.Decode(l)
+}
+
+// ForEach calls handler for each package ref in list
+func (l *PackageRefList) ForEach(handler func([]byte) error) error {
+	var err error
+	for _, p := range l.Refs {
+		err = handler(p)
+		if err != nil {
+			return err
+		}
+	}
+	return err
+}
+
+// Substract returns all packages in l that are not in r
+func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
+	result := &PackageRefList{Refs: make([][]byte, 0, 128)}
+
+	// pointer to left and right reflists
+	il, ir := 0, 0
+	// length of reflists
+	ll, lr := l.Len(), r.Len()
+
+	for il < ll || ir < lr {
+		if il == ll {
+			// left list exhausted, we got the result
+			break
+		}
+		if ir == lr {
+			// right list exhausted, append what is left to result
+			result.Refs = append(result.Refs, l.Refs[il:]...)
+			break
+		}
+
+		rel := bytes.Compare(l.Refs[il], r.Refs[ir])
+		if rel == 0 {
+			// r contains entry from l, so we skip it
+			il++
+			ir++
+		} else if rel < 0 {
+			// item il is not in r, append
+			result.Refs = append(result.Refs, l.Refs[il])
+			il++
+		} else {
+			// skip over to next item in r
+			ir++
+		}
+	}
+
+	return result
+}
+
+// PackageDiff is a difference between two packages in a list.
+//
+// If left & right are present, difference is in package version
+// If left is nil, package is present only in right
+// If right is nil, package is present only in left
+type PackageDiff struct {
+	Left, Right *Package
+}
+
+// PackageDiffs is a list of PackageDiff records
+type PackageDiffs []PackageDiff
+
+// Diff calculates difference between two reflists
+func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageCollection) (result PackageDiffs, err error) {
+	result = make(PackageDiffs, 0, 128)
+
+	// pointer to left and right reflists
+	il, ir := 0, 0
+	// length of reflists
+	ll, lr := l.Len(), r.Len()
+	// cached loaded packages on the left & right
+	pl, pr := (*Package)(nil), (*Package)(nil)
+
+	// until we reached end of both lists
+	for il < ll || ir < lr {
+		// if we've exhausted left list, pull the rest from the right
+		if il == ll {
+			pr, err = packageCollection.ByKey(r.Refs[ir])
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, PackageDiff{Left: nil, Right: pr})
+			ir++
+			continue
+		}
+		// if we've exhausted right list, pull the rest from the left
+		if ir == lr {
+			pl, err = packageCollection.ByKey(l.Refs[il])
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, PackageDiff{Left: pl, Right: nil})
+			il++
+			continue
+		}
+
+		// refs on both sides are present, load them
+		rl, rr := l.Refs[il], r.Refs[ir]
+		// compare refs
+		rel := bytes.Compare(rl, rr)
+
+		if rel == 0 {
+			// refs are identical, so are packages, advance pointer
+			il++
+			ir++
+			pl, pr = nil, nil
+		} else {
+			// load pl & pr if they haven't been loaded before
+			if pl == nil {
+				pl, err = packageCollection.ByKey(rl)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			if pr == nil {
+				pr, err = packageCollection.ByKey(rr)
+				if err != nil {
+					return nil, err
+				}
+			}
+
+			// is pl & pr the same package, but different version?
+			if pl.Name == pr.Name && pl.Architecture == pr.Architecture {
+				result = append(result, PackageDiff{Left: pl, Right: pr})
+				il++
+				ir++
+				pl, pr = nil, nil
+			} else {
+				// otherwise pl or pr is missing on one of the sides
+				if rel < 0 {
+					result = append(result, PackageDiff{Left: pl, Right: nil})
+					il++
+					pl = nil
+				} else {
+					result = append(result, PackageDiff{Left: nil, Right: pr})
+					ir++
+					pr = nil
+				}
+			}
+
+		}
+	}
+
+	return
+}
+
+// Merge merges reflist r into current reflist. If overrideMatching, merge replaces matching packages (by architecture/name)
+// with reference from r, otherwise all packages are saved.
+func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching bool) (result *PackageRefList) {
+	// pointer to left and right reflists
+	il, ir := 0, 0
+	// length of reflists
+	ll, lr := l.Len(), r.Len()
+
+	result = &PackageRefList{}
+	result.Refs = make([][]byte, 0, ll+lr)
+
+	// until we reached end of both lists
+	for il < ll || ir < lr {
+		// if we've exhausted left list, pull the rest from the right
+		if il == ll {
+			result.Refs = append(result.Refs, r.Refs[ir:]...)
+			break
+		}
+		// if we've exhausted right list, pull the rest from the left
+		if ir == lr {
+			result.Refs = append(result.Refs, l.Refs[il:]...)
+			break
+		}
+
+		// refs on both sides are present, load them
+		rl, rr := l.Refs[il], r.Refs[ir]
+		// compare refs
+		rel := bytes.Compare(rl, rr)
+
+		if rel == 0 {
+			// refs are identical, so are packages, advance pointer
+			result.Refs = append(result.Refs, l.Refs[il])
+			il++
+			ir++
+		} else {
+			if overrideMatching {
+				partsL := bytes.Split(rl, []byte(" "))
+				archL, nameL := partsL[0][1:], partsL[1]
+
+				partsR := bytes.Split(rr, []byte(" "))
+				archR, nameR := partsR[0][1:], partsR[1]
+
+				if bytes.Compare(archL, archR) == 0 && bytes.Compare(nameL, nameR) == 0 {
+					// override with package from the right
+					result.Refs = append(result.Refs, r.Refs[ir])
+					il++
+					ir++
+					continue
+				}
+			}
+
+			// otherwise append smallest of two
+			if rel < 0 {
+				result.Refs = append(result.Refs, l.Refs[il])
+				il++
+			} else {
+				result.Refs = append(result.Refs, r.Refs[ir])
+				ir++
+			}
+
+		}
+	}
+
+	return
+}

debian/reflist_test.go

@@ -0,0 +1,275 @@
+package debian
+
+import (
+	"errors"
+	"github.com/smira/aptly/database"
+	. "launchpad.net/gocheck"
+)
+
+type PackageRefListSuite struct {
+	// Simple list with "real" packages from stanzas
+	list                   *PackageList
+	p1, p2, p3, p4, p5, p6 *Package
+}
+
+var _ = Suite(&PackageRefListSuite{})
+
+func (s *PackageRefListSuite) SetUpTest(c *C) {
+	s.list = NewPackageList()
+
+	s.p1 = NewPackageFromControlFile(packageStanza.Copy())
+	s.p2 = NewPackageFromControlFile(packageStanza.Copy())
+	stanza := packageStanza.Copy()
+	stanza["Package"] = "mars-invaders"
+	s.p3 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Size"] = "42"
+	s.p4 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Package"] = "lonely-strangers"
+	s.p5 = NewPackageFromControlFile(stanza)
+	stanza = packageStanza.Copy()
+	stanza["Version"] = "99.1"
+	s.p6 = NewPackageFromControlFile(stanza)
+}
+
+func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
+	db, _ := database.OpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+	coll.Update(s.p1)
+	coll.Update(s.p3)
+
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
+
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	_, err := NewPackageListFromRefList(reflist, coll, nil)
+	c.Assert(err, ErrorMatches, "unable to load package with key.*")
+
+	coll.Update(s.p5)
+	coll.Update(s.p6)
+
+	list, err := NewPackageListFromRefList(reflist, coll, nil)
+	c.Assert(err, IsNil)
+	c.Check(list.Len(), Equals, 4)
+	c.Check(list.Add(s.p4), ErrorMatches, "conflict in package.*")
+
+	list, err = NewPackageListFromRefList(nil, coll, nil)
+	c.Assert(err, IsNil)
+	c.Check(list.Len(), Equals, 0)
+}
+
+func (s *PackageRefListSuite) TestNewPackageRefList(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
+
+	reflist := NewPackageRefListFromPackageList(s.list)
+	c.Assert(reflist.Len(), Equals, 4)
+	c.Check(reflist.Refs[0], DeepEquals, []byte(s.p1.Key("")))
+	c.Check(reflist.Refs[1], DeepEquals, []byte(s.p6.Key("")))
+	c.Check(reflist.Refs[2], DeepEquals, []byte(s.p5.Key("")))
+	c.Check(reflist.Refs[3], DeepEquals, []byte(s.p3.Key("")))
+
+	reflist = NewPackageRefList()
+	c.Check(reflist.Len(), Equals, 0)
+}
+
+func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
+
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	reflist2 := &PackageRefList{}
+	err := reflist2.Decode(reflist.Encode())
+	c.Assert(err, IsNil)
+	c.Check(reflist2.Len(), Equals, reflist.Len())
+	c.Check(reflist2.Refs, DeepEquals, reflist.Refs)
+}
+
+func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	s.list.Add(s.p6)
+
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	Len := 0
+	err := reflist.ForEach(func([]byte) error {
+		Len++
+		return nil
+	})
+
+	c.Check(Len, Equals, 4)
+	c.Check(err, IsNil)
+
+	e := errors.New("b")
+
+	err = reflist.ForEach(func([]byte) error {
+		return e
+	})
+
+	c.Check(err, Equals, e)
+}
+
+func (s *PackageRefListSuite) TestSubstract(c *C) {
+	r1 := []byte("r1")
+	r2 := []byte("r2")
+	r3 := []byte("r3")
+	r4 := []byte("r4")
+	r5 := []byte("r5")
+
+	empty := &PackageRefList{Refs: [][]byte{}}
+	l1 := &PackageRefList{Refs: [][]byte{r1, r2, r3, r4}}
+	l2 := &PackageRefList{Refs: [][]byte{r1, r3}}
+	l3 := &PackageRefList{Refs: [][]byte{r2, r4}}
+	l4 := &PackageRefList{Refs: [][]byte{r4, r5}}
+	l5 := &PackageRefList{Refs: [][]byte{r1, r2, r3}}
+
+	c.Check(l1.Substract(empty), DeepEquals, l1)
+	c.Check(l1.Substract(l2), DeepEquals, l3)
+	c.Check(l1.Substract(l3), DeepEquals, l2)
+	c.Check(l1.Substract(l4), DeepEquals, l5)
+	c.Check(empty.Substract(l1), DeepEquals, empty)
+	c.Check(l2.Substract(l3), DeepEquals, l2)
+}
+
+func (s *PackageRefListSuite) TestDiff(c *C) {
+	db, _ := database.OpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+
+	packages := []*Package{
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		&Package{Name: "xyz", Version: "3.0", Architecture: "sparc"},     //6
+	}
+
+	for _, p := range packages {
+		coll.Update(p)
+	}
+
+	listA := NewPackageList()
+	listA.Add(packages[0])
+	listA.Add(packages[1])
+	listA.Add(packages[2])
+	listA.Add(packages[3])
+	listA.Add(packages[6])
+
+	listB := NewPackageList()
+	listB.Add(packages[0])
+	listB.Add(packages[2])
+	listB.Add(packages[4])
+	listB.Add(packages[5])
+
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
+
+	diffAA, err := reflistA.Diff(reflistA, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAA, HasLen, 0)
+
+	diffAB, err := reflistA.Diff(reflistB, coll)
+	c.Check(err, IsNil)
+	c.Check(diffAB, HasLen, 4)
+
+	c.Check(diffAB[0].Left, IsNil)
+	c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
+
+	c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
+
+	c.Check(diffAB[2].Left.String(), Equals, "dpkg_1.7_i386")
+	c.Check(diffAB[2].Right, IsNil)
+
+	c.Check(diffAB[3].Left.String(), Equals, "xyz_3.0_sparc")
+	c.Check(diffAB[3].Right, IsNil)
+
+	diffBA, err := reflistB.Diff(reflistA, coll)
+	c.Check(err, IsNil)
+	c.Check(diffBA, HasLen, 4)
+
+	c.Check(diffBA[0].Right, IsNil)
+	c.Check(diffBA[0].Left.String(), Equals, "app_1.1~bp2_amd64")
+
+	c.Check(diffBA[1].Right.String(), Equals, "app_1.1~bp1_i386")
+	c.Check(diffBA[1].Left.String(), Equals, "app_1.1~bp2_i386")
+
+	c.Check(diffBA[2].Right.String(), Equals, "dpkg_1.7_i386")
+	c.Check(diffBA[2].Left, IsNil)
+
+	c.Check(diffBA[3].Right.String(), Equals, "xyz_3.0_sparc")
+	c.Check(diffBA[3].Left, IsNil)
+
+}
+
+func (s *PackageRefListSuite) TestMerge(c *C) {
+	db, _ := database.OpenDB(c.MkDir())
+	coll := NewPackageCollection(db)
+
+	packages := []*Package{
+		&Package{Name: "lib", Version: "1.0", Architecture: "i386"},      //0
+		&Package{Name: "dpkg", Version: "1.7", Architecture: "i386"},     //1
+		&Package{Name: "data", Version: "1.1~bp1", Architecture: "all"},  //2
+		&Package{Name: "app", Version: "1.1~bp1", Architecture: "i386"},  //3
+		&Package{Name: "app", Version: "1.1~bp2", Architecture: "i386"},  //4
+		&Package{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //5
+		&Package{Name: "dpkg", Version: "1.0", Architecture: "i386"},     //6
+		&Package{Name: "xyz", Version: "1.0", Architecture: "sparc"},     //7
+	}
+
+	for _, p := range packages {
+		coll.Update(p)
+	}
+
+	listA := NewPackageList()
+	listA.Add(packages[0])
+	listA.Add(packages[1])
+	listA.Add(packages[2])
+	listA.Add(packages[3])
+	listA.Add(packages[7])
+
+	listB := NewPackageList()
+	listB.Add(packages[0])
+	listB.Add(packages[2])
+	listB.Add(packages[4])
+	listB.Add(packages[5])
+	listB.Add(packages[6])
+
+	reflistA := NewPackageRefListFromPackageList(listA)
+	reflistB := NewPackageRefListFromPackageList(listB)
+
+	toStrSlice := func(reflist *PackageRefList) (result []string) {
+		result = make([]string, reflist.Len())
+		for i, r := range reflist.Refs {
+			result[i] = string(r)
+		}
+		return
+	}
+
+	mergeAB := reflistA.Merge(reflistB, true)
+	mergeBA := reflistB.Merge(reflistA, true)
+
+	c.Check(toStrSlice(mergeAB), DeepEquals,
+		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+	c.Check(toStrSlice(mergeBA), DeepEquals,
+		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+
+	mergeABall := reflistA.Merge(reflistB, false)
+	mergeBAall := reflistB.Merge(reflistA, false)
+
+	c.Check(mergeABall, DeepEquals, mergeBAall)
+	c.Check(toStrSlice(mergeBAall), DeepEquals,
+		[]string{"Pall data 1.1~bp1", "Pamd64 app 1.1~bp2", "Pi386 app 1.1~bp1", "Pi386 app 1.1~bp2", "Pi386 dpkg 1.0", "Pi386 dpkg 1.7", "Pi386 lib 1.0", "Psparc xyz 1.0"})
+}

debian/remote.go

@@ -1,15 +1,18 @@
-// Package debian implements Debian-specific repository handling
 package debian
 
 import (
 	"bytes"
 	"code.google.com/p/go-uuid/uuid"
 	"fmt"
+	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
 	"net/url"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -17,7 +20,6 @@ import (
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
 // Repostitory could be filtered when fetching by components, architectures
-// TODO: support flat format
 type RemoteRepo struct {
 	// Permanent internal ID
 	UUID string
@@ -31,10 +33,14 @@ type RemoteRepo struct {
 	Components []string
 	// List of architectures to fetch, if empty, then fetch all architectures
 	Architectures []string
+	// Should we download sources?
+	DownloadSources bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
 	LastDownloadDate time.Time
+	// Checksums for release files
+	ReleaseFiles map[string]utils.ChecksumInfo
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -42,35 +48,67 @@ type RemoteRepo struct {
 }
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
-func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, architectures []string) (*RemoteRepo, error) {
+func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
+	architectures []string, downloadSources bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:          uuid.New(),
-		Name:          name,
-		ArchiveRoot:   archiveRoot,
-		Distribution:  distribution,
-		Components:    components,
-		Architectures: architectures,
+		UUID:            uuid.New(),
+		Name:            name,
+		ArchiveRoot:     archiveRoot,
+		Distribution:    distribution,
+		Components:      components,
+		Architectures:   architectures,
+		DownloadSources: downloadSources,
 	}
 
 	err := result.prepare()
 	if err != nil {
 		return nil, err
 	}
+
+	if result.Distribution == "." || result.Distribution == "./" {
+		// flat repo
+		result.Distribution = ""
+		result.Architectures = nil
+		if len(result.Components) > 0 {
+			return nil, fmt.Errorf("components aren't supported for flat repos")
+		}
+		result.Components = nil
+	}
+
 	return result, nil
 }
 
 func (repo *RemoteRepo) prepare() error {
 	var err error
+
+	// Add final / to URL
+	if !strings.HasSuffix(repo.ArchiveRoot, "/") {
+		repo.ArchiveRoot = repo.ArchiveRoot + "/"
+	}
+
 	repo.archiveRootURL, err = url.Parse(repo.ArchiveRoot)
 	return err
 }
 
 // String interface
 func (repo *RemoteRepo) String() string {
-	return fmt.Sprintf("[%s]: %s %s", repo.Name, repo.ArchiveRoot, repo.Distribution)
+	srcFlag := ""
+	if repo.DownloadSources {
+		srcFlag = " [src]"
+	}
+	distribution := repo.Distribution
+	if distribution == "" {
+		distribution = "./"
+	}
+	return fmt.Sprintf("[%s]: %s %s%s", repo.Name, repo.ArchiveRoot, distribution, srcFlag)
 }
 
-// NumPackages return number of packages retrived from remore repo
+// IsFlat determines if repository is flat
+func (repo *RemoteRepo) IsFlat() bool {
+	return repo.Distribution == ""
+}
+
+// NumPackages return number of packages retrived from remote repo
 func (repo *RemoteRepo) NumPackages() int {
 	if repo.packageRefs == nil {
 		return 0
@@ -78,20 +116,49 @@ func (repo *RemoteRepo) NumPackages() int {
 	return repo.packageRefs.Len()
 }
 
-// ReleaseURL returns URL to Release file in repo root
-// TODO: InRelease, Release.gz, Release.bz2 handling
-func (repo *RemoteRepo) ReleaseURL() *url.URL {
-	path := &url.URL{Path: fmt.Sprintf("dists/%s/Release", repo.Distribution)}
+// RefList returns package list for repo
+func (repo *RemoteRepo) RefList() *PackageRefList {
+	return repo.packageRefs
+}
+
+// ReleaseURL returns URL to Release* files in repo root
+func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
+	var path *url.URL
+
+	if !repo.IsFlat() {
+		path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
+	} else {
+		path = &url.URL{Path: name}
+	}
+
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
-// BinaryURL returns URL of Packages file for given component and
+// FlatBinaryURL returns URL to Packages files for flat repo
+func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
+	path := &url.URL{Path: "Packages"}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
+// FlatSourcesURL returns URL to Sources files for flat repo
+func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
+	path := &url.URL{Path: "Sources"}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
+// BinaryURL returns URL of Packages files for given component and
 // architecture
 func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
 	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// SourcesURL returns URL of Sources files for given component
+func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
+	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/source/Sources", repo.Distribution, component)}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -100,12 +167,64 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
 }
 
 // Fetch updates information about repository
-func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
-	// Download release file to temporary URL
-	release, err := utils.DownloadTemp(d, repo.ReleaseURL().String())
-	if err != nil {
-		return err
+func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
+	var (
+		release *os.File
+		err     error
+	)
+
+	if verifier == nil {
+		// 0. Just download release file to temporary URL
+		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		if err != nil {
+			return err
+		}
+	} else {
+		// 1. try InRelease file
+		inrelease, err := http.DownloadTemp(d, repo.ReleaseURL("InRelease").String())
+		if err != nil {
+			goto splitsignature
+		}
+		defer inrelease.Close()
+
+		err = verifier.VerifyClearsigned(inrelease)
+		if err != nil {
+			goto splitsignature
+		}
+
+		inrelease.Seek(0, 0)
+
+		release, err = verifier.ExtractClearsigned(inrelease)
+		if err != nil {
+			goto splitsignature
+		}
+
+		goto ok
+
+	splitsignature:
+		// 2. try Release + Release.gpg
+		release, err = http.DownloadTemp(d, repo.ReleaseURL("Release").String())
+		if err != nil {
+			return err
+		}
+
+		releasesig, err := http.DownloadTemp(d, repo.ReleaseURL("Release.gpg").String())
+		if err != nil {
+			return err
+		}
+
+		err = verifier.VerifyDetachedSignature(releasesig, release)
+		if err != nil {
+			return err
+		}
+
+		_, err = release.Seek(0, 0)
+		if err != nil {
+			return err
+		}
 	}
+ok:
+
 	defer release.Close()
 
 	sreader := NewControlFileReader(release)
@@ -114,119 +233,204 @@ func (repo *RemoteRepo) Fetch(d utils.Downloader) error {
 		return err
 	}
 
-	architectures := strings.Split(stanza["Architectures"], " ")
-	if len(repo.Architectures) == 0 {
-		repo.Architectures = architectures
-	} else {
-		err = utils.StringsIsSubset(repo.Architectures, architectures,
-			fmt.Sprintf("architecture %%s not available in repo %s", repo))
-		if err != nil {
-			return err
+	if !repo.IsFlat() {
+		architectures := strings.Split(stanza["Architectures"], " ")
+		if len(repo.Architectures) == 0 {
+			repo.Architectures = architectures
+		} else {
+			err = utils.StringsIsSubset(repo.Architectures, architectures,
+				fmt.Sprintf("architecture %%s not available in repo %s", repo))
+			if err != nil {
+				return err
+			}
+		}
+
+		components := strings.Split(stanza["Components"], " ")
+		if len(repo.Components) == 0 {
+			repo.Components = components
+		} else {
+			err = utils.StringsIsSubset(repo.Components, components,
+				fmt.Sprintf("component %%s not available in repo %s", repo))
+			if err != nil {
+				return err
+			}
 		}
 	}
 
-	components := strings.Split(stanza["Components"], " ")
-	if len(repo.Components) == 0 {
-		repo.Components = components
-	} else {
-		err = utils.StringsIsSubset(repo.Components, components,
-			fmt.Sprintf("component %%s not available in repo %s", repo))
-		if err != nil {
-			return err
+	repo.ReleaseFiles = make(map[string]utils.ChecksumInfo)
+
+	parseSums := func(field string, setter func(sum *utils.ChecksumInfo, data string)) error {
+		for _, line := range strings.Split(stanza[field], "\n") {
+			line = strings.TrimSpace(line)
+			if line == "" {
+				continue
+			}
+			parts := strings.Fields(line)
+
+			if len(parts) != 3 {
+				return fmt.Errorf("unparseable hash sum line: %#v", line)
+			}
+
+			size, err := strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return fmt.Errorf("unable to parse size: %s", err)
+			}
+
+			sum := repo.ReleaseFiles[parts[2]]
+
+			sum.Size = size
+			setter(&sum, parts[0])
+
+			repo.ReleaseFiles[parts[2]] = sum
 		}
+
+		delete(stanza, field)
+
+		return nil
+	}
+
+	err = parseSums("MD5Sum", func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	if err != nil {
+		return err
+	}
+
+	err = parseSums("SHA1", func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	if err != nil {
+		return err
+	}
+
+	err = parseSums("SHA256", func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	if err != nil {
+		return err
 	}
 
-	delete(stanza, "MD5Sum")
-	delete(stanza, "SHA1")
-	delete(stanza, "SHA256")
 	repo.Meta = stanza
 
 	return nil
 }
 
 // Download downloads all repo files
-func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageCollection, packageRepo *Repository) error {
+func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, packageCollection *PackageCollection, packagePool aptly.PackagePool, ignoreMismatch bool) error {
 	list := NewPackageList()
 
-	fmt.Printf("Downloading & parsing package files...\n")
+	progress.Printf("Downloading & parsing package files...\n")
 
-	// Download and parse all Release files
-	for _, component := range repo.Components {
-		for _, architecture := range repo.Architectures {
-			packagesReader, packagesFile, err := utils.DownloadTryCompression(d, repo.BinaryURL(component, architecture).String())
-			if err != nil {
-				return err
+	// Download and parse all Packages & Source files
+	packagesURLs := [][]string{}
+
+	if repo.IsFlat() {
+		packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
+		if repo.DownloadSources {
+			packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
+		}
+	} else {
+		for _, component := range repo.Components {
+			for _, architecture := range repo.Architectures {
+				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
 			}
-			defer packagesFile.Close()
-
-			sreader := NewControlFileReader(packagesReader)
-
-			for {
-				stanza, err := sreader.ReadStanza()
-				if err != nil {
-					return err
-				}
-				if stanza == nil {
-					break
-				}
-
-				p := NewPackageFromControlFile(stanza)
-
-				list.Add(p)
+			if repo.DownloadSources {
+				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
 			}
 		}
 	}
 
-	fmt.Printf("Saving packages to database...\n")
+	for _, info := range packagesURLs {
+		url, kind := info[0], info[1]
+		packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch)
+		if err != nil {
+			return err
+		}
+		defer packagesFile.Close()
 
-	// Save package meta information to DB
-	err := list.ForEach(func(p *Package) error {
-		return packageCollection.Update(p)
-	})
+		stat, _ := packagesFile.Stat()
+		progress.InitBar(stat.Size(), true)
 
-	if err != nil {
-		return fmt.Errorf("unable to save packages to db: %s", err)
+		sreader := NewControlFileReader(packagesReader)
+
+		for {
+			stanza, err := sreader.ReadStanza()
+			if err != nil {
+				return err
+			}
+			if stanza == nil {
+				break
+			}
+
+			off, _ := packagesFile.Seek(0, 1)
+			progress.SetBar(int(off))
+
+			var p *Package
+
+			if kind == "binary" {
+				p = NewPackageFromControlFile(stanza)
+			} else if kind == "source" {
+				p, err = NewSourcePackageFromControlFile(stanza)
+				if err != nil {
+					return err
+				}
+			}
+			err = list.Add(p)
+			if err != nil {
+				return err
+			}
+
+			err = packageCollection.Update(p)
+			if err != nil {
+				return err
+			}
+		}
+
+		progress.ShutdownBar()
 	}
 
-	fmt.Printf("Building download queue...\n")
+	progress.Printf("Building download queue...\n")
 
 	// Build download queue
 	queued := make(map[string]PackageDownloadTask, list.Len())
 	count := 0
 	downloadSize := int64(0)
 
-	err = list.ForEach(func(p *Package) error {
-		list, err := p.DownloadList(packageRepo)
+	err := list.ForEach(func(p *Package) error {
+		list, err := p.DownloadList(packagePool)
 		if err != nil {
 			return err
 		}
+		p.files = nil
 
 		for _, task := range list {
 			key := task.RepoURI + "-" + task.DestinationPath
 			_, found := queued[key]
 			if !found {
 				count++
-				downloadSize += task.Size
+				downloadSize += task.Checksums.Size
 				queued[key] = task
 			}
 		}
 
 		return nil
 	})
-
 	if err != nil {
 		return fmt.Errorf("unable to build download queue: %s", err)
 	}
 
-	fmt.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))
+	repo.packageRefs = NewPackageRefListFromPackageList(list)
+	// free up package list, we don't need it after this point
+	list = nil
+
+	progress.Printf("Download queue: %d items, %.2f GiB size\n", count, float64(downloadSize)/(1024.0*1024.0*1024.0))
+
+	progress.InitBar(downloadSize, true)
 
 	// Download all package files
 	ch := make(chan error, len(queued))
 
 	for _, task := range queued {
-		d.Download(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch)
+		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
 	}
 
+	// We don't need queued after this point
+	queued = nil
+
 	// Wait for all downloads to finish
 	errors := make([]string, 0)
 
@@ -238,12 +442,13 @@ func (repo *RemoteRepo) Download(d utils.Downloader, packageCollection *PackageC
 		count--
 	}
 
+	progress.ShutdownBar()
+
 	if len(errors) > 0 {
 		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
 	}
 
 	repo.LastDownloadDate = time.Now()
-	repo.packageRefs = NewPackageRefListFromPackageList(list)
 
 	return nil
 }
@@ -387,3 +592,29 @@ func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error)
 func (collection *RemoteRepoCollection) Len() int {
 	return len(collection.list)
 }
+
+// Drop removes remote repo from collection
+func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
+	repoPosition := -1
+
+	for i, r := range collection.list {
+		if r == repo {
+			repoPosition = i
+			break
+		}
+	}
+
+	if repoPosition == -1 {
+		panic("repo not found!")
+	}
+
+	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
+		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+
+	err := collection.db.Delete(repo.Key())
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Delete(repo.RefKey())
+}

debian/remote_test.go

@@ -2,15 +2,43 @@ package debian
 
 import (
 	"errors"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/files"
+	"github.com/smira/aptly/http"
 	"github.com/smira/aptly/utils"
+	"io"
+	"io/ioutil"
 	. "launchpad.net/gocheck"
-	"testing"
+	"os"
 )
 
-// Launch gocheck tests
-func Test(t *testing.T) {
-	TestingT(t)
+type NullVerifier struct {
+}
+
+func (n *NullVerifier) InitKeyring() error {
+	return nil
+}
+
+func (n *NullVerifier) AddKeyring(keyring string) {
+}
+
+func (n *NullVerifier) VerifyDetachedSignature(signature, cleartext io.Reader) error {
+	return nil
+}
+
+func (n *NullVerifier) VerifyClearsigned(clearsigned io.Reader) error {
+	return nil
+}
+
+func (n *NullVerifier) ExtractClearsigned(clearsigned io.Reader) (text *os.File, err error) {
+	text, _ = ioutil.TempFile("", "aptly-test")
+	io.Copy(text, clearsigned)
+	text.Seek(0, 0)
+	os.Remove(text.Name())
+
+	return
 }
 
 type PackageListMixinSuite struct {
@@ -40,68 +68,150 @@ func (s *PackageListMixinSuite) SetUpPackages() {
 type RemoteRepoSuite struct {
 	PackageListMixinSuite
 	repo              *RemoteRepo
-	downloader        *utils.FakeDownloader
+	flat              *RemoteRepo
+	downloader        *http.FakeDownloader
+	progress          aptly.Progress
 	db                database.Storage
 	packageCollection *PackageCollection
-	packageRepo       *Repository
+	packagePool       aptly.PackagePool
 }
 
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
-	s.downloader = utils.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false)
+	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	s.progress = console.NewProgress()
 	s.db, _ = database.OpenDB(c.MkDir())
 	s.packageCollection = NewPackageCollection(s.db)
-	s.packageRepo = NewRepository(c.MkDir())
+	s.packagePool = files.NewPackagePool(c.MkDir())
 	s.SetUpPackages()
+	s.progress.Start()
 }
 
 func (s *RemoteRepoSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
 	s.db.Close()
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{})
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false)
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
+func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
+	c.Check(s.flat.Distribution, Equals, "")
+	c.Check(s.flat.Architectures, IsNil)
+	c.Check(s.flat.Components, IsNil)
+
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false)
+	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
+}
+
+func (s *RemoteRepoSuite) TestString(c *C) {
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze")
+	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./")
+
+	s.repo.DownloadSources = true
+	s.flat.DownloadSources = true
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src]")
+	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
+}
+
 func (s *RemoteRepoSuite) TestNumPackages(c *C) {
 	c.Check(s.repo.NumPackages(), Equals, 0)
 	s.repo.packageRefs = s.reflist
 	c.Check(s.repo.NumPackages(), Equals, 3)
 }
 
+func (s *RemoteRepoSuite) TestIsFlat(c *C) {
+	c.Check(s.repo.IsFlat(), Equals, false)
+	c.Check(s.flat.IsFlat(), Equals, true)
+}
+
+func (s *RemoteRepoSuite) TestRefList(c *C) {
+	s.repo.packageRefs = s.reflist
+	c.Check(s.repo.RefList(), Equals, s.reflist)
+}
+
 func (s *RemoteRepoSuite) TestReleaseURL(c *C) {
-	c.Assert(s.repo.ReleaseURL().String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/Release")
+	c.Assert(s.repo.ReleaseURL("Release").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/Release")
+	c.Assert(s.repo.ReleaseURL("InRelease").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/InRelease")
+
+	c.Assert(s.flat.ReleaseURL("Release").String(), Equals, "http://repos.express42.com/virool/precise/Release")
 }
 
 func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
 	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
 }
 
+func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
+	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
+}
+
+func (s *RemoteRepoSuite) TestFlatBinaryURL(c *C) {
+	c.Assert(s.flat.FlatBinaryURL().String(), Equals, "http://repos.express42.com/virool/precise/Packages")
+}
+
+func (s *RemoteRepoSuite) TestFlatSourcesURL(c *C) {
+	c.Assert(s.flat.FlatSourcesURL().String(), Equals, "http://repos.express42.com/virool/precise/Sources")
+}
+
 func (s *RemoteRepoSuite) TestPackageURL(c *C) {
 	c.Assert(s.repo.PackageURL("pool/main/0/0ad/0ad_0~r11863-2_i386.deb").String(), Equals,
 		"http://mirror.yandex.ru/debian/pool/main/0/0ad/0ad_0~r11863-2_i386.deb")
 }
 
 func (s *RemoteRepoSuite) TestFetch(c *C) {
-	err := s.repo.Fetch(s.downloader)
+	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, IsNil)
 	c.Assert(s.repo.Architectures, DeepEquals, []string{"amd64", "armel", "armhf", "i386", "powerpc"})
 	c.Assert(s.repo.Components, DeepEquals, []string{"main"})
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	c.Check(s.repo.ReleaseFiles, HasLen, 39)
+	c.Check(s.repo.ReleaseFiles["main/binary-i386/Packages.bz2"], DeepEquals,
+		utils.ChecksumInfo{
+			Size:   734,
+			MD5:    "7954ed80936429687122b554620c1b5b",
+			SHA1:   "95a463a0739bf9ff622c8d68f6e4598d400f5248",
+			SHA256: "377890a26f99db55e117dfc691972dcbbb7d8be1630c8fc8297530c205377f2b"})
+}
+
+func (s *RemoteRepoSuite) TestFetchNullVerifier1(c *C) {
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/InRelease", errors.New("404"))
+	downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
+	downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release.gpg", "GPG")
+
+	err := s.repo.Fetch(downloader, &NullVerifier{})
+	c.Assert(err, IsNil)
+	c.Assert(s.repo.Architectures, DeepEquals, []string{"amd64", "armel", "armhf", "i386", "powerpc"})
+	c.Assert(s.repo.Components, DeepEquals, []string{"main"})
+	c.Assert(downloader.Empty(), Equals, true)
+}
+
+func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/InRelease", exampleReleaseFile)
+
+	err := s.repo.Fetch(downloader, &NullVerifier{})
+	c.Assert(err, IsNil)
+	c.Assert(s.repo.Architectures, DeepEquals, []string{"amd64", "armel", "armhf", "i386", "powerpc"})
+	c.Assert(s.repo.Components, DeepEquals, []string{"main"})
+	c.Assert(downloader.Empty(), Equals, true)
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"})
-	err := s.repo.Fetch(s.downloader)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false)
+	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"})
-	err := s.repo.Fetch(s.downloader)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false)
+	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
 
@@ -128,7 +238,7 @@ func (s *RemoteRepoSuite) TestRefKey(c *C) {
 func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.repo.Architectures = []string{"i386"}
 
-	err := s.repo.Fetch(s.downloader)
+	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, IsNil)
 
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
@@ -136,7 +246,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.downloader, s.packageCollection, s.packageRepo)
+	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 	c.Assert(s.repo.packageRefs, NotNil)
@@ -144,13 +254,124 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packageRepo)
+	result, err := pkg.VerifyFiles(s.packagePool)
 	c.Check(result, Equals, true)
 	c.Check(err, IsNil)
 
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
+func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
+	s.repo.Architectures = []string{"i386"}
+	s.repo.DownloadSources = true
+
+	err := s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", errors.New("HTTP 404"))
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", errors.New("HTTP 404"))
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
+	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
+	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
+	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
+	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
+
+	err = s.repo.Download(s.progress, s.downloader, s.packageCollection, s.packagePool, false)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	pkg, err := s.packageCollection.ByKey(s.repo.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	result, err := pkg.VerifyFiles(s.packagePool)
+	c.Check(result, Equals, true)
+	c.Check(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+
+	pkg, err = s.packageCollection.ByKey(s.repo.packageRefs.Refs[1])
+	c.Assert(err, IsNil)
+
+	result, err = pkg.VerifyFiles(s.packagePool)
+	c.Check(result, Equals, true)
+	c.Check(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "access-modifier-checker")
+}
+
+func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
+
+	err := s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	result, err := pkg.VerifyFiles(s.packagePool)
+	c.Check(result, Equals, true)
+	c.Check(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+}
+
+func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
+	s.flat.DownloadSources = true
+
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Release", exampleReleaseFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", errors.New("HTTP 404"))
+	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", errors.New("HTTP 404"))
+	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
+	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
+	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
+	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
+	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
+
+	err := s.flat.Fetch(downloader, nil)
+	c.Assert(err, IsNil)
+
+	err = s.flat.Download(s.progress, downloader, s.packageCollection, s.packagePool, false)
+	c.Assert(err, IsNil)
+	c.Assert(downloader.Empty(), Equals, true)
+	c.Assert(s.flat.packageRefs, NotNil)
+
+	pkg, err := s.packageCollection.ByKey(s.flat.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	result, err := pkg.VerifyFiles(s.packagePool)
+	c.Check(result, Equals, true)
+	c.Check(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+
+	pkg, err = s.packageCollection.ByKey(s.flat.packageRefs.Refs[1])
+	c.Assert(err, IsNil)
+
+	result, err = pkg.VerifyFiles(s.packagePool)
+	c.Check(result, Equals, true)
+	c.Check(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "access-modifier-checker")
+}
+
 type RemoteRepoCollectionSuite struct {
 	PackageListMixinSuite
 	db         database.Storage
@@ -173,7 +394,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	r, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -191,7 +412,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	r, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err = s.collection.ByUUID(repo.UUID)
@@ -200,7 +421,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -221,7 +442,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -242,6 +463,32 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 	c.Assert(err, Equals, e)
 }
 
+func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.collection.Add(repo1)
+
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false)
+	s.collection.Add(repo2)
+
+	r1, _ := s.collection.ByUUID(repo1.UUID)
+	c.Check(r1, Equals, repo1)
+
+	err := s.collection.Drop(repo1)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByUUID(repo1.UUID)
+	c.Check(err, ErrorMatches, "mirror .* not found")
+
+	collection := NewRemoteRepoCollection(s.db)
+	_, err = collection.ByName("yandex")
+	c.Check(err, ErrorMatches, "mirror .* not found")
+
+	r2, _ := collection.ByName("tyndex")
+	c.Check(r2.String(), Equals, repo2.String())
+
+	c.Check(func() { s.collection.Drop(repo1) }, Panics, "repo not found!")
+}
+
 const exampleReleaseFile = `Origin: LP-PPA-agenda-developers-daily
 Label: Agenda Daily Builds
 Suite: precise
@@ -264,7 +511,7 @@ MD5Sum:
  c63d31e8e3a5650c29a7124e541d6c23              134 main/binary-armhf/Release
  4059d198768f9f8dc9372dc1c54bc3c3               14 main/binary-armhf/Packages.bz2
  d41d8cd98f00b204e9800998ecf8427e                0 main/binary-armhf/Packages
- 708fc548e709eea0dfd2d7edb6098829             1344 main/binary-i386/Packages
+ c8d336856df67d509032bb54145c2f89              826 main/binary-i386/Packages
  92262f0668b265401291f0467bc93763              133 main/binary-i386/Release
  7954ed80936429687122b554620c1b5b              734 main/binary-i386/Packages.bz2
  e2eef4fe7d285b12c511adfa3a39069e              641 main/binary-i386/Packages.gz
@@ -288,7 +535,7 @@ MD5Sum:
  4059d198768f9f8dc9372dc1c54bc3c3               14 main/debian-installer/binary-powerpc/Packages.bz2
  9d10bb61e59bd799891ae4fbcf447ec9               29 main/debian-installer/binary-powerpc/Packages.gz
  3481d65651306df1596dca9078c2506a              135 main/source/Release
- 0531474bd4630bfcfd39048be830483d             1119 main/source/Sources
+ 0459b7e4512db5479cb982bac6e2f9a1             2003 main/source/Sources
  3d83a489f1bd3c04226aa6520b8a6d07              656 main/source/Sources.bz2
  b062b5b77094aeeb05ca8dbb1ecf68a9              592 main/source/Sources.gz
 SHA1:
@@ -304,7 +551,7 @@ SHA1:
  585a452e27c2e7e047c49d4b0a7459d8c627aa08              134 main/binary-armhf/Release
  64a543afbb5f4bf728636bdcbbe7a2ed0804adc2               14 main/binary-armhf/Packages.bz2
  da39a3ee5e6b4b0d3255bfef95601890afd80709                0 main/binary-armhf/Packages
- 2bfad956c2d2437924a8527970858c59823451b7             1344 main/binary-i386/Packages
+ 1d2f0cd7a3c9e687b853eb277e241cd712b6e3b1              826 main/binary-i386/Packages
  16020809662f9bda36eb516d0995658dd94d1ad5              133 main/binary-i386/Release
  95a463a0739bf9ff622c8d68f6e4598d400f5248              734 main/binary-i386/Packages.bz2
  bf8c0dec9665ba78311c97cae1755d4b2e60af76              641 main/binary-i386/Packages.gz
@@ -328,7 +575,7 @@ SHA1:
  64a543afbb5f4bf728636bdcbbe7a2ed0804adc2               14 main/debian-installer/binary-powerpc/Packages.bz2
  3df6ca52b6e8ecfb4a8fac6b8e02c777e3c7960d               29 main/debian-installer/binary-powerpc/Packages.gz
  49cfec0c9b1df3a25e983a3ddf29d15b0e376e02              135 main/source/Release
- 4987db83999b0a8bbbbeeb183f066cadb87a5fa5             1119 main/source/Sources
+ 6b92e0fc84307226172696fde59ca5f33f380b57             2003 main/source/Sources
  ecb8afea11030a5df46941cb8ec297ca24c85736              656 main/source/Sources.bz2
  923e71383969c91146f12fa8cd121397f2467a2e              592 main/source/Sources.gz
 SHA256:
@@ -344,7 +591,7 @@ SHA256:
  d25382b633c4a1621f8df6ce86e5c63da2e506a377e05ae9453238bb18191540              134 main/binary-armhf/Release
  d3dda84eb03b9738d118eb2be78e246106900493c0ae07819ad60815134a8058               14 main/binary-armhf/Packages.bz2
  e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855                0 main/binary-armhf/Packages
- 9cd4bad3462e795bad509a44bae48622f2e9c9e56aafc999419cc5221f087dc8             1344 main/binary-i386/Packages
+ b1bb341bb613363ca29440c2eb9c08a9289de5458209990ec502ed27711a83a2              826 main/binary-i386/Packages
  e5aaceaac5ecb59143a4b4ed2bf700fe85d6cf08addd10cf2058bde697b7b219              133 main/binary-i386/Release
  377890a26f99db55e117dfc691972dcbbb7d8be1630c8fc8297530c205377f2b              734 main/binary-i386/Packages.bz2
  6361e8efc67d2e7c1a8db45388aec0311007c0a1bd96698623ddeb5ed0bdc914              641 main/binary-i386/Packages.gz
@@ -368,7 +615,7 @@ SHA256:
  d3dda84eb03b9738d118eb2be78e246106900493c0ae07819ad60815134a8058               14 main/debian-installer/binary-powerpc/Packages.bz2
  825d493158fe0f50ca1acd70367aefa391170563af2e4ee9cedbcbe6796c8384               29 main/debian-installer/binary-powerpc/Packages.gz
  d683102993b6f11067ce86d73111f067e36a199e9dc1f4295c8b19c274dc9ef8              135 main/source/Release
- a8707486566f1623f0e50c0f8f61d93a93d79fb3043b6e1c407fc9f2afb002ce             1119 main/source/Sources
+ 45f868fd5d9efe611d67572ffcf96a00a5b9ec38ea5102753290c38c36b8c282             2003 main/source/Sources
  d178f1e310218d9f0f16c37d0780637f1cf3640a94a7fb0e24dc940c51b1e115              656 main/source/Sources.bz2
  080228b550da407fb8ac73fb30b37323468fd2b2de98dd56a324ee7d701f6103              592 main/source/Sources.gz`
 
@@ -388,7 +635,9 @@ Section: utils
 Priority: optional
 Filename: pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb
 Size: 3
-MD5sum: cdc997dc06126e18ea9ba843efed9811
-SHA1: 049ba341d520c447fa2e6a1f8c871b3dbbe00106
-SHA256: 4487115ca47fe9acd95355b9278f30e18c53f33c385252252d3d7948d650d1d0
+MD5sum: d16fb36f0911f878998c136191af705e
+SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
+SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
 `
+
+const exampleSourcesFile = sourcePackageMeta

debian/repository.go

@@ -1,93 +0,0 @@
-package debian
-
-import (
-	"fmt"
-	"github.com/smira/aptly/utils"
-	"os"
-	"path/filepath"
-)
-
-// Repository directory structure:
-// <root>
-// \- pool
-//    \- ab
-//       \- ae
-//          \- package.deb
-// \- public
-//    \- dists
-//       \- squeeze
-//          \- Release
-//          \- main
-//             \- binary-i386
-//                \- Packages.bz2
-//                   references packages from pool
-//    \- pool
-//       contains symlinks to main pool
-
-// Repository abstract file system with package pool and published package repos
-type Repository struct {
-	RootPath string
-}
-
-// NewRepository creates new instance of repository which specified root
-func NewRepository(root string) *Repository {
-	return &Repository{RootPath: root}
-}
-
-// PoolPath returns full path to package file in pool givan any name and hash of file contents
-func (r *Repository) PoolPath(filename string, hashMD5 string) (string, error) {
-	filename = filepath.Base(filename)
-	if filename == "." || filename == "/" {
-		return "", fmt.Errorf("filename %s is invalid", filename)
-	}
-
-	return filepath.Join(r.RootPath, "pool", hashMD5[0:2], hashMD5[2:4], filename), nil
-}
-
-// PublicPath returns root of public part of repository
-func (r *Repository) PublicPath() string {
-	return filepath.Join(r.RootPath, "public")
-}
-
-// MkDir creates directory recursively under public path
-func (r *Repository) MkDir(path string) error {
-	return os.MkdirAll(filepath.Join(r.RootPath, "public", path), 0755)
-}
-
-// CreateFile creates file for writing under public path
-func (r *Repository) CreateFile(path string) (*os.File, error) {
-	return os.Create(filepath.Join(r.RootPath, "public", path))
-}
-
-// RemoveDirs removes directory structure under public path
-func (r *Repository) RemoveDirs(path string) error {
-	filepath := filepath.Join(r.RootPath, "public", path)
-	fmt.Printf("Removing %s...\n", filepath)
-	return os.RemoveAll(filepath)
-}
-
-// LinkFromPool links package file from pool to dist's pool location
-func (r *Repository) LinkFromPool(prefix string, component string, sourcePath string, poolDirectory string) (string, error) {
-	baseName := filepath.Base(sourcePath)
-
-	relPath := filepath.Join("pool", component, poolDirectory, baseName)
-	poolPath := filepath.Join(r.RootPath, "public", prefix, "pool", component, poolDirectory)
-
-	err := os.MkdirAll(poolPath, 0755)
-	if err != nil {
-		return "", err
-	}
-
-	_, err = os.Stat(filepath.Join(poolPath, baseName))
-	if err == nil { // already exists, skip
-		return relPath, nil
-	}
-
-	err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
-	return relPath, err
-}
-
-// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
-func (r *Repository) ChecksumsForFile(path string) (*utils.ChecksumInfo, error) {
-	return utils.ChecksumsForFile(filepath.Join(r.RootPath, "public", path))
-}

debian/snapshot.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/smira/aptly/database"
+	"github.com/smira/aptly/utils"
 	"github.com/ugorji/go/codec"
 	"log"
 	"time"
@@ -46,6 +47,23 @@ func NewSnapshotFromRepository(name string, repo *RemoteRepo) (*Snapshot, error)
 	}, nil
 }
 
+// NewSnapshotFromLocalRepo creates snapshot from current state of local repository
+func NewSnapshotFromLocalRepo(name string, repo *LocalRepo) (*Snapshot, error) {
+	if repo.packageRefs == nil {
+		return nil, errors.New("local repo doesn't have packages")
+	}
+
+	return &Snapshot{
+		UUID:        uuid.New(),
+		Name:        name,
+		CreatedAt:   time.Now(),
+		SourceKind:  "local",
+		SourceIDs:   []string{repo.UUID},
+		Description: fmt.Sprintf("Snapshot from local repo %s", repo),
+		packageRefs: repo.packageRefs,
+	}, nil
+}
+
 // NewSnapshotFromPackageList creates snapshot from PackageList
 func NewSnapshotFromPackageList(name string, sources []*Snapshot, list *PackageList, description string) *Snapshot {
 	return NewSnapshotFromRefList(name, sources, NewPackageRefListFromPackageList(list), description)
@@ -194,6 +212,42 @@ func (collection *SnapshotCollection) ByUUID(uuid string) (*Snapshot, error) {
 	return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
 }
 
+// ByRemoteRepoSource looks up snapshots that have specified RemoteRepo as a source
+func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Snapshot {
+	result := make([]*Snapshot, 0)
+
+	for _, s := range collection.list {
+		if s.SourceKind == "repo" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+			result = append(result, s)
+		}
+	}
+	return result
+}
+
+// ByLocalRepoSource looks up snapshots that have specified LocalRepo as a source
+func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snapshot {
+	result := make([]*Snapshot, 0)
+
+	for _, s := range collection.list {
+		if s.SourceKind == "local" && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
+			result = append(result, s)
+		}
+	}
+	return result
+}
+
+// BySnapshotSource looks up snapshots that have specified snapshot as a source
+func (collection *SnapshotCollection) BySnapshotSource(snapshot *Snapshot) []*Snapshot {
+	result := make([]*Snapshot, 0)
+
+	for _, s := range collection.list {
+		if s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID) {
+			result = append(result, s)
+		}
+	}
+	return result
+}
+
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) error {
 	var err error
@@ -211,3 +265,29 @@ func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) err
 func (collection *SnapshotCollection) Len() int {
 	return len(collection.list)
 }
+
+// Drop removes snapshot from collection
+func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
+	snapshotPosition := -1
+
+	for i, s := range collection.list {
+		if s == snapshot {
+			snapshotPosition = i
+			break
+		}
+	}
+
+	if snapshotPosition == -1 {
+		panic("snapshot not found!")
+	}
+
+	collection.list[len(collection.list)-1], collection.list[snapshotPosition], collection.list =
+		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+
+	err := collection.db.Delete(snapshot.Key())
+	if err != nil {
+		return err
+	}
+
+	return collection.db.Delete(snapshot.RefKey())
+}

debian/snapshot_test.go

@@ -15,7 +15,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -32,6 +32,22 @@ func (s *SnapshotSuite) TestNewSnapshotFromRepository(c *C) {
 	c.Check(err, ErrorMatches, ".*not updated")
 }
 
+func (s *SnapshotSuite) TestNewSnapshotFromLocalRepo(c *C) {
+	localRepo := NewLocalRepo("lala", "hoorah!")
+
+	_, err := NewSnapshotFromLocalRepo("snap2", localRepo)
+	c.Check(err, ErrorMatches, "local repo doesn't have packages")
+
+	localRepo.UpdateRefList(s.reflist)
+	snapshot, _ := NewSnapshotFromLocalRepo("snap1", localRepo)
+	c.Check(snapshot.Name, Equals, "snap1")
+	c.Check(snapshot.NumPackages(), Equals, 3)
+	c.Check(snapshot.RefList().Len(), Equals, 3)
+	c.Check(snapshot.SourceKind, Equals, "local")
+	c.Check(snapshot.SourceIDs, DeepEquals, []string{localRepo.UUID})
+
+}
+
 func (s *SnapshotSuite) TestNewSnapshotFromPackageList(c *C) {
 	snap, _ := NewSnapshotFromRepository("snap1", s.repo)
 
@@ -79,7 +95,9 @@ type SnapshotCollectionSuite struct {
 	PackageListMixinSuite
 	db                   database.Storage
 	repo1, repo2         *RemoteRepo
+	lrepo1, lrepo2       *LocalRepo
 	snapshot1, snapshot2 *Snapshot
+	snapshot3, snapshot4 *Snapshot
 	collection           *SnapshotCollection
 }
 
@@ -90,13 +108,21 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{})
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{})
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
+
+	s.lrepo1 = NewLocalRepo("local1", "")
+	s.lrepo1.packageRefs = s.reflist
+	s.snapshot3, _ = NewSnapshotFromLocalRepo("snap3", s.lrepo1)
+
+	s.lrepo2 = NewLocalRepo("local2", "")
+	s.lrepo2.packageRefs = s.reflist
+	s.snapshot4, _ = NewSnapshotFromLocalRepo("snap4", s.lrepo2)
 }
 
 func (s *SnapshotCollectionSuite) TearDownTest(c *C) {
@@ -158,3 +184,66 @@ func (s *SnapshotCollectionSuite) TestForEachAndLen(c *C) {
 	})
 	c.Assert(err, Equals, e)
 }
+
+func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
+
+	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
+	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
+
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+
+	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot{})
+}
+
+func (s *SnapshotCollectionSuite) TestFindByLocalRepoSource(c *C) {
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
+	c.Assert(s.collection.Add(s.snapshot3), IsNil)
+	c.Assert(s.collection.Add(s.snapshot4), IsNil)
+
+	c.Check(s.collection.ByLocalRepoSource(s.lrepo1), DeepEquals, []*Snapshot{s.snapshot3})
+	c.Check(s.collection.ByLocalRepoSource(s.lrepo2), DeepEquals, []*Snapshot{s.snapshot4})
+
+	lrepo3 := NewLocalRepo("other", "")
+
+	c.Check(s.collection.ByLocalRepoSource(lrepo3), DeepEquals, []*Snapshot{})
+}
+
+func (s *SnapshotCollectionSuite) TestFindSnapshotSource(c *C) {
+	snapshot3 := NewSnapshotFromRefList("snap3", []*Snapshot{s.snapshot1, s.snapshot2}, s.reflist, "desc1")
+	snapshot4 := NewSnapshotFromRefList("snap4", []*Snapshot{s.snapshot1}, s.reflist, "desc2")
+	snapshot5 := NewSnapshotFromRefList("snap5", []*Snapshot{snapshot3}, s.reflist, "desc3")
+
+	c.Assert(s.collection.Add(s.snapshot1), IsNil)
+	c.Assert(s.collection.Add(s.snapshot2), IsNil)
+	c.Assert(s.collection.Add(snapshot3), IsNil)
+	c.Assert(s.collection.Add(snapshot4), IsNil)
+	c.Assert(s.collection.Add(snapshot5), IsNil)
+
+	c.Check(s.collection.BySnapshotSource(s.snapshot1), DeepEquals, []*Snapshot{snapshot3, snapshot4})
+	c.Check(s.collection.BySnapshotSource(s.snapshot2), DeepEquals, []*Snapshot{snapshot3})
+	c.Check(s.collection.BySnapshotSource(snapshot5), DeepEquals, []*Snapshot{})
+}
+
+func (s *SnapshotCollectionSuite) TestDrop(c *C) {
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot2)
+
+	snap, _ := s.collection.ByUUID(s.snapshot1.UUID)
+	c.Check(snap, Equals, s.snapshot1)
+
+	err := s.collection.Drop(s.snapshot1)
+	c.Check(err, IsNil)
+
+	_, err = s.collection.ByUUID(s.snapshot1.UUID)
+	c.Check(err, ErrorMatches, "snapshot .* not found")
+
+	collection := NewSnapshotCollection(s.db)
+
+	_, err = collection.ByUUID(s.snapshot1.UUID)
+	c.Check(err, ErrorMatches, "snapshot .* not found")
+
+	c.Check(func() { s.collection.Drop(s.snapshot1) }, Panics, "snapshot not found!")
+}

debian/version.go

@@ -228,8 +228,19 @@ func ParseDependencyVariants(variants string) (l []Dependency, err error) {
 	return
 }
 
-// ParseDependency parses dependency in format "pkg (>= 1.35)" into parts
+// ParseDependency parses dependency in format "pkg (>= 1.35) [arch]" into parts
 func ParseDependency(dep string) (d Dependency, err error) {
+	if strings.HasSuffix(dep, "}") {
+		i := strings.LastIndex(dep, "{")
+		if i == -1 {
+			err = fmt.Errorf("unable to parse dependency: %s", dep)
+			return
+		}
+		d.Architecture = dep[i+1 : len(dep)-1]
+
+		dep = strings.TrimSpace(dep[:i])
+	}
+
 	if !strings.HasSuffix(dep, ")") {
 		d.Pkg = strings.TrimSpace(dep)
 		d.Relation = VersionDontCare
@@ -244,12 +255,17 @@ func ParseDependency(dep string) (d Dependency, err error) {
 
 	d.Pkg = strings.TrimSpace(dep[0:i])
 
-	rel := dep[i+1 : i+2]
-	if dep[i+2] == '>' || dep[i+2] == '<' || dep[i+2] == '=' {
-		rel += dep[i+2 : i+3]
-		d.Version = strings.TrimSpace(dep[i+3 : len(dep)-1])
+	rel := ""
+	if dep[i+1] == '>' || dep[i+1] == '<' || dep[i+1] == '=' {
+		rel += dep[i+1 : i+2]
+		if dep[i+2] == '>' || dep[i+2] == '<' || dep[i+2] == '=' {
+			rel += dep[i+2 : i+3]
+			d.Version = strings.TrimSpace(dep[i+3 : len(dep)-1])
+		} else {
+			d.Version = strings.TrimSpace(dep[i+2 : len(dep)-1])
+		}
 	} else {
-		d.Version = strings.TrimSpace(dep[i+2 : len(dep)-1])
+		d.Version = strings.TrimSpace(dep[i+1 : len(dep)-1])
 	}
 
 	switch rel {
@@ -261,10 +277,10 @@ func ParseDependency(dep string) (d Dependency, err error) {
 		d.Relation = VersionLess
 	case ">>":
 		d.Relation = VersionGreater
-	case "=":
+	case "", "=":
 		d.Relation = VersionEqual
 	default:
-		err = fmt.Errorf("relation unknown: %s", rel)
+		err = fmt.Errorf("relation unknown %#v in dependency %s", rel, dep)
 	}
 
 	return

debian/version_test.go

@@ -104,6 +104,7 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 	c.Check(d.Pkg, Equals, "dpkg")
 	c.Check(d.Relation, Equals, VersionGreaterOrEqual)
 	c.Check(d.Version, Equals, "1.6")
+	c.Check(d.Architecture, Equals, "")
 
 	d, e = ParseDependency("dpkg(>>1.6)")
 	c.Check(e, IsNil)
@@ -111,6 +112,18 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 	c.Check(d.Relation, Equals, VersionGreater)
 	c.Check(d.Version, Equals, "1.6")
 
+	d, e = ParseDependency("dpkg(1.6)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "dpkg")
+	c.Check(d.Relation, Equals, VersionEqual)
+	c.Check(d.Version, Equals, "1.6")
+
+	d, e = ParseDependency("dpkg ( 1.6)")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "dpkg")
+	c.Check(d.Relation, Equals, VersionEqual)
+	c.Check(d.Version, Equals, "1.6")
+
 	d, e = ParseDependency("dpkg (> 1.6)")
 	c.Check(e, IsNil)
 	c.Check(d.Pkg, Equals, "dpkg")
@@ -141,6 +154,20 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 	c.Check(d.Relation, Equals, VersionGreater)
 	c.Check(d.Version, Equals, "1.6")
 
+	d, e = ParseDependency("dpkg (>>1.6) {i386}")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "dpkg")
+	c.Check(d.Relation, Equals, VersionGreater)
+	c.Check(d.Version, Equals, "1.6")
+	c.Check(d.Architecture, Equals, "i386")
+
+	d, e = ParseDependency("dpkg{i386}")
+	c.Check(e, IsNil)
+	c.Check(d.Pkg, Equals, "dpkg")
+	c.Check(d.Relation, Equals, VersionDontCare)
+	c.Check(d.Version, Equals, "")
+	c.Check(d.Architecture, Equals, "i386")
+
 	d, e = ParseDependency("dpkg ")
 	c.Check(e, IsNil)
 	c.Check(d.Pkg, Equals, "dpkg")
@@ -152,6 +179,12 @@ func (s *VersionSuite) TestParseDependency(c *C) {
 
 	d, e = ParseDependency("dpkg==1.6)")
 	c.Check(e, ErrorMatches, "unable to parse.*")
+
+	d, e = ParseDependency("dpkg i386}")
+	c.Check(e, ErrorMatches, "unable to parse.*")
+
+	d, e = ParseDependency("dpkg ) {i386}")
+	c.Check(e, ErrorMatches, "unable to parse.*")
 }
 
 func (s *VersionSuite) TestParseDependencyVariants(c *C) {

files/files.go

@@ -0,0 +1,19 @@
+// Package files handles operation on filesystem for both public pool and published files
+package files
+
+// Repository directory structure:
+// <root>
+// \- pool
+//    \- ab
+//       \- ae
+//          \- package.deb
+// \- public
+//    \- dists
+//       \- squeeze
+//          \- Release
+//          \- main
+//             \- binary-i386
+//                \- Packages.bz2
+//                   references packages from pool
+//    \- pool
+//       contains symlinks to main pool

files/files_test.go

@@ -0,0 +1,11 @@
+package files
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

files/package_pool.go

@@ -0,0 +1,154 @@
+package files
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// PackagePool is deduplicated storage of package files on filesystem
+type PackagePool struct {
+	rootPath string
+}
+
+// Check interface
+var (
+	_ aptly.PackagePool = (*PackagePool)(nil)
+)
+
+// NewPackagePool creates new instance of PackagePool which specified root
+func NewPackagePool(root string) *PackagePool {
+	return &PackagePool{rootPath: filepath.Join(root, "pool")}
+}
+
+// RelativePath returns path relative to pool's root for package files given MD5 and original filename
+func (pool *PackagePool) RelativePath(filename string, hashMD5 string) (string, error) {
+	filename = filepath.Base(filename)
+	if filename == "." || filename == "/" {
+		return "", fmt.Errorf("filename %s is invalid", filename)
+	}
+
+	if len(hashMD5) < 4 {
+		return "", fmt.Errorf("unable to compute pool location for filename %v, MD5 is missing", filename)
+	}
+
+	return filepath.Join(hashMD5[0:2], hashMD5[2:4], filename), nil
+}
+
+// Path returns full path to package file in pool given any name and hash of file contents
+func (pool *PackagePool) Path(filename string, hashMD5 string) (string, error) {
+	relative, err := pool.RelativePath(filename, hashMD5)
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(pool.rootPath, relative), nil
+}
+
+// FilepathList returns file paths of all the files in the pool
+func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
+	dirs, err := ioutil.ReadDir(pool.rootPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	if len(dirs) == 0 {
+		return nil, nil
+	}
+
+	if progress != nil {
+		progress.InitBar(int64(len(dirs)), false)
+		defer progress.ShutdownBar()
+	}
+
+	result := []string{}
+
+	for _, dir := range dirs {
+		err = filepath.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if !info.IsDir() {
+				result = append(result, path[len(pool.rootPath)+1:])
+			}
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		if progress != nil {
+			progress.AddBar(1)
+		}
+	}
+
+	return result, nil
+}
+
+// Remove deletes file in package pool returns its size
+func (pool *PackagePool) Remove(path string) (size int64, err error) {
+	path = filepath.Join(pool.rootPath, path)
+
+	info, err := os.Stat(path)
+	if err != nil {
+		return 0, err
+	}
+
+	err = os.Remove(path)
+	return info.Size(), err
+}
+
+// Import copies file into package pool
+func (pool *PackagePool) Import(path string, hashMD5 string) error {
+	source, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer source.Close()
+
+	sourceInfo, err := source.Stat()
+	if err != nil {
+		return err
+	}
+
+	poolPath, err := pool.Path(path, hashMD5)
+	if err != nil {
+		return err
+	}
+
+	targetInfo, err := os.Stat(poolPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			// unable to stat target location?
+			return err
+		}
+		// file doesn't exist, that's ok
+	} else {
+		if targetInfo.Size() != sourceInfo.Size() {
+			// trying to overwrite file?
+			return fmt.Errorf("unable to import into pool: file %s already exists", poolPath)
+		}
+	}
+
+	// create subdirs as necessary
+	err = os.MkdirAll(filepath.Dir(poolPath), 0755)
+	if err != nil {
+		return err
+	}
+
+	target, err := os.Create(poolPath)
+	if err != nil {
+		return err
+	}
+	defer target.Close()
+
+	_, err = io.Copy(target, source)
+
+	return err
+}

files/package_pool_test.go

@@ -0,0 +1,119 @@
+package files
+
+import (
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+type PackagePoolSuite struct {
+	pool *PackagePool
+}
+
+var _ = Suite(&PackagePoolSuite{})
+
+func (s *PackagePoolSuite) SetUpTest(c *C) {
+	s.pool = NewPackagePool(c.MkDir())
+
+}
+
+func (s *PackagePoolSuite) TestRelativePath(c *C) {
+	path, err := s.pool.RelativePath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
+	c.Assert(err, IsNil)
+	c.Assert(path, Equals, "91/b1/package.deb")
+
+	_, err = s.pool.RelativePath("/", "91b1a1480b90b9e269ca44d897b12575")
+	c.Assert(err, ErrorMatches, ".*is invalid")
+	_, err = s.pool.RelativePath("", "91b1a1480b90b9e269ca44d897b12575")
+	c.Assert(err, ErrorMatches, ".*is invalid")
+	_, err = s.pool.RelativePath("a/b/package.deb", "9")
+	c.Assert(err, ErrorMatches, ".*MD5 is missing")
+}
+
+func (s *PackagePoolSuite) TestPath(c *C) {
+	path, err := s.pool.Path("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
+	c.Assert(err, IsNil)
+	c.Assert(path, Equals, filepath.Join(s.pool.rootPath, "91/b1/package.deb"))
+
+	_, err = s.pool.Path("/", "91b1a1480b90b9e269ca44d897b12575")
+	c.Assert(err, ErrorMatches, ".*is invalid")
+}
+
+func (s *PackagePoolSuite) TestFilepathList(c *C) {
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, IsNil)
+
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{})
+
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), nil, 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), nil, 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), nil, 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), nil, 0644)
+
+	list, err = s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "ae/0c/2.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
+}
+
+func (s *PackagePoolSuite) TestRemove(c *C) {
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0b"), 0755)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "bd", "0a"), 0755)
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "ae", "0c"), 0755)
+
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "1.deb"), []byte("1"), 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "ae", "0c", "2.deb"), []byte("22"), 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0a", "3.deb"), []byte("333"), 0644)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "bd", "0b", "4.deb"), []byte("4444"), 0644)
+
+	size, err := s.pool.Remove("ae/0c/2.deb")
+	c.Check(err, IsNil)
+	c.Check(size, Equals, int64(2))
+
+	_, err = s.pool.Remove("ae/0c/2.deb")
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+
+	list, err := s.pool.FilepathList(nil)
+	c.Check(err, IsNil)
+	c.Check(list, DeepEquals, []string{"ae/0c/1.deb", "bd/0a/3.deb", "bd/0b/4.deb"})
+}
+
+func (s *PackagePoolSuite) TestImportOk(c *C) {
+	_, _File, _, _ := runtime.Caller(0)
+	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
+	c.Check(err, IsNil)
+
+	info, err := os.Stat(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"))
+	c.Check(err, IsNil)
+	c.Check(info.Size(), Equals, int64(2738))
+
+	// double import, should be ok
+	err = s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
+	c.Check(err, IsNil)
+}
+
+func (s *PackagePoolSuite) TestImportNotExist(c *C) {
+	err := s.pool.Import("no-such-file", "91b1a1480b90b9e269ca44d897b12575")
+	c.Check(err, ErrorMatches, ".*no such file or directory")
+}
+
+func (s *PackagePoolSuite) TestImportOverwrite(c *C) {
+	_, _File, _, _ := runtime.Caller(0)
+	debFile := filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
+
+	os.MkdirAll(filepath.Join(s.pool.rootPath, "91", "b1"), 0755)
+	ioutil.WriteFile(filepath.Join(s.pool.rootPath, "91", "b1", "libboost-program-options-dev_1.49.0.1_i386.deb"), []byte("1"), 0644)
+
+	err := s.pool.Import(debFile, "91b1a1480b90b9e269ca44d897b12575")
+	c.Check(err, ErrorMatches, "unable to import into pool.*")
+}

files/public.go

@@ -0,0 +1,83 @@
+package files
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+)
+
+// PublishedStorage abstract file system with public dirs (published repos)
+type PublishedStorage struct {
+	rootPath string
+}
+
+// Check interface
+var (
+	_ aptly.PublishedStorage = (*PublishedStorage)(nil)
+)
+
+// NewPublishedStorage creates new instance of PublishedStorage which specified root
+func NewPublishedStorage(root string) *PublishedStorage {
+	return &PublishedStorage{rootPath: filepath.Join(root, "public")}
+}
+
+// PublicPath returns root of public part
+func (storage *PublishedStorage) PublicPath() string {
+	return storage.rootPath
+}
+
+// MkDir creates directory recursively under public path
+func (storage *PublishedStorage) MkDir(path string) error {
+	return os.MkdirAll(filepath.Join(storage.rootPath, path), 0755)
+}
+
+// CreateFile creates file for writing under public path
+func (storage *PublishedStorage) CreateFile(path string) (*os.File, error) {
+	return os.Create(filepath.Join(storage.rootPath, path))
+}
+
+// RemoveDirs removes directory structure under public path
+func (storage *PublishedStorage) RemoveDirs(path string) error {
+	filepath := filepath.Join(storage.rootPath, path)
+	fmt.Printf("Removing %s...\n", filepath)
+	return os.RemoveAll(filepath)
+}
+
+// LinkFromPool links package file from pool to dist's pool location
+//
+// prefix is publishing prefix for this repo (e.g. empty or "ppa/")
+// component is component name when publishing (e.g. main)
+// poolDirectory is desired location in pool (like liba/libav/)
+// sourcePool is instance of aptly.PackagePool
+// sourcePath is filepath to package file in package pool
+//
+// LinkFromPool returns relative path for the published file to be included in package index
+func (storage *PublishedStorage) LinkFromPool(prefix string, component string, poolDirectory string, sourcePool aptly.PackagePool, sourcePath string) (string, error) {
+	// verify that package pool is local pool is filesystem pool
+	_ = sourcePool.(*PackagePool)
+
+	baseName := filepath.Base(sourcePath)
+
+	relPath := filepath.Join("pool", component, poolDirectory, baseName)
+	poolPath := filepath.Join(storage.rootPath, prefix, "pool", component, poolDirectory)
+
+	err := os.MkdirAll(poolPath, 0755)
+	if err != nil {
+		return "", err
+	}
+
+	_, err = os.Stat(filepath.Join(poolPath, baseName))
+	if err == nil { // already exists, skip
+		return relPath, nil
+	}
+
+	err = os.Link(sourcePath, filepath.Join(poolPath, baseName))
+	return relPath, err
+}
+
+// ChecksumsForFile proxies requests to utils.ChecksumsForFile, joining public path
+func (storage *PublishedStorage) ChecksumsForFile(path string) (utils.ChecksumInfo, error) {
+	return utils.ChecksumsForFile(filepath.Join(storage.rootPath, path))
+}

files/public_test.go

@@ -1,73 +1,65 @@
-package debian
+package files
 
 import (
+	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
 	"path/filepath"
 	"syscall"
 )
 
-type RepositorySuite struct {
-	repo *Repository
+type PublishedStorageSuite struct {
+	root    string
+	storage *PublishedStorage
 }
 
-var _ = Suite(&RepositorySuite{})
+var _ = Suite(&PublishedStorageSuite{})
 
-func (s *RepositorySuite) SetUpTest(c *C) {
-	s.repo = NewRepository(c.MkDir())
+func (s *PublishedStorageSuite) SetUpTest(c *C) {
+	s.root = c.MkDir()
+	s.storage = NewPublishedStorage(s.root)
 }
 
-func (s *RepositorySuite) TestPoolPath(c *C) {
-	path, err := s.repo.PoolPath("a/b/package.deb", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, IsNil)
-	c.Assert(path, Equals, filepath.Join(s.repo.RootPath, "pool", "91/b1/package.deb"))
-
-	_, err = s.repo.PoolPath("/", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, ErrorMatches, ".*is invalid")
-	_, err = s.repo.PoolPath("", "91b1a1480b90b9e269ca44d897b12575")
-	c.Assert(err, ErrorMatches, ".*is invalid")
+func (s *PublishedStorageSuite) TestPublicPath(c *C) {
+	c.Assert(s.storage.PublicPath(), Equals, filepath.Join(s.root, "public"))
 }
 
-func (s *RepositorySuite) TestPublicPath(c *C) {
-	c.Assert(s.repo.PublicPath(), Equals, filepath.Join(s.repo.RootPath, "public"))
-}
-
-func (s *RepositorySuite) TestMkDir(c *C) {
-	err := s.repo.MkDir("ppa/dists/squeeze/")
+func (s *PublishedStorageSuite) TestMkDir(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/"))
+	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/"))
 	c.Assert(err, IsNil)
 }
 
-func (s *RepositorySuite) TestCreateFile(c *C) {
-	err := s.repo.MkDir("ppa/dists/squeeze/")
+func (s *PublishedStorageSuite) TestCreateFile(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.repo.CreateFile("ppa/dists/squeeze/Release")
+	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
 	c.Assert(err, IsNil)
 	defer file.Close()
 
-	_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/Release"))
+	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 }
 
-func (s *RepositorySuite) TestRemoveDirs(c *C) {
-	err := s.repo.MkDir("ppa/dists/squeeze/")
+func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
 
-	file, err := s.repo.CreateFile("ppa/dists/squeeze/Release")
+	file, err := s.storage.CreateFile("ppa/dists/squeeze/Release")
 	c.Assert(err, IsNil)
 	defer file.Close()
 
-	err = s.repo.RemoveDirs("ppa/dists/")
+	err = s.storage.RemoveDirs("ppa/dists/")
 
-	_, err = os.Stat(filepath.Join(s.repo.RootPath, "public/ppa/dists/squeeze/Release"))
+	_, err = os.Stat(filepath.Join(s.storage.rootPath, "ppa/dists/squeeze/Release"))
 	c.Assert(err, NotNil)
 	c.Assert(os.IsNotExist(err), Equals, true)
 }
 
-func (s *RepositorySuite) TestLinkFromPool(c *C) {
+func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	tests := []struct {
 		prefix           string
 		component        string
@@ -105,23 +97,22 @@ func (s *RepositorySuite) TestLinkFromPool(c *C) {
 		},
 	}
 
+	pool := NewPackagePool(s.root)
+
 	for _, t := range tests {
-		t.sourcePath = filepath.Join(s.repo.RootPath, t.sourcePath)
+		t.sourcePath = filepath.Join(s.root, t.sourcePath)
 
 		err := os.MkdirAll(filepath.Dir(t.sourcePath), 0755)
 		c.Assert(err, IsNil)
 
-		file, err := os.Create(t.sourcePath)
+		err = ioutil.WriteFile(t.sourcePath, []byte("Contents"), 0644)
 		c.Assert(err, IsNil)
 
-		file.Write([]byte("Contents"))
-		file.Close()
-
-		path, err := s.repo.LinkFromPool(t.prefix, t.component, t.sourcePath, t.poolDirectory)
+		path, err := s.storage.LinkFromPool(t.prefix, t.component, t.poolDirectory, pool, t.sourcePath)
 		c.Assert(err, IsNil)
 		c.Assert(path, Equals, t.expectedFilename)
 
-		st, err := os.Stat(filepath.Join(s.repo.RootPath, "public", t.prefix, t.expectedFilename))
+		st, err := os.Stat(filepath.Join(s.storage.rootPath, t.prefix, t.expectedFilename))
 		c.Assert(err, IsNil)
 
 		info := st.Sys().(*syscall.Stat_t)

http/download.go

@@ -1,56 +1,55 @@
-package utils
+package http
 
 import (
 	"compress/bzip2"
 	"compress/gzip"
 	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
 	"path/filepath"
+	"strings"
 )
 
-// Downloader is parallel HTTP fetcher
-type Downloader interface {
-	Download(url string, destination string, result chan<- error)
-	Pause()
-	Resume()
-	Shutdown()
-}
-
 // Check interface
 var (
-	_ Downloader = &downloaderImpl{}
+	_ aptly.Downloader = (*downloaderImpl)(nil)
 )
 
 // downloaderImpl is implementation of Downloader interface
 type downloaderImpl struct {
-	queue   chan *downloadTask
-	stop    chan bool
-	stopped chan bool
-	pause   chan bool
-	unpause chan bool
-	threads int
+	queue    chan *downloadTask
+	stop     chan bool
+	stopped  chan bool
+	pause    chan bool
+	unpause  chan bool
+	progress aptly.Progress
+	threads  int
 }
 
 // downloadTask represents single item in queue
 type downloadTask struct {
-	url         string
-	destination string
-	result      chan<- error
+	url            string
+	destination    string
+	result         chan<- error
+	expected       utils.ChecksumInfo
+	ignoreMismatch bool
 }
 
 // NewDownloader creates new instance of Downloader which specified number
 // of threads
-func NewDownloader(threads int) Downloader {
+func NewDownloader(threads int, progress aptly.Progress) aptly.Downloader {
 	downloader := &downloaderImpl{
-		queue:   make(chan *downloadTask, 1000),
-		stop:    make(chan bool),
-		stopped: make(chan bool),
-		pause:   make(chan bool),
-		unpause: make(chan bool),
-		threads: threads,
+		queue:    make(chan *downloadTask, 1000),
+		stop:     make(chan bool),
+		stopped:  make(chan bool),
+		pause:    make(chan bool),
+		unpause:  make(chan bool),
+		threads:  threads,
+		progress: progress,
 	}
 
 	for i := 0; i < downloader.threads; i++ {
@@ -86,14 +85,25 @@ func (downloader *downloaderImpl) Resume() {
 	}
 }
 
+// GetProgress returns Progress object
+func (downloader *downloaderImpl) GetProgress() aptly.Progress {
+	return downloader.progress
+}
+
 // Download starts new download task
 func (downloader *downloaderImpl) Download(url string, destination string, result chan<- error) {
-	downloader.queue <- &downloadTask{url: url, destination: destination, result: result}
+	downloader.DownloadWithChecksum(url, destination, result, utils.ChecksumInfo{Size: -1}, false)
+}
+
+// DownloadWithChecksum starts new download task with checksum verification
+func (downloader *downloaderImpl) DownloadWithChecksum(url string, destination string, result chan<- error,
+	expected utils.ChecksumInfo, ignoreMismatch bool) {
+	downloader.queue <- &downloadTask{url: url, destination: destination, result: result, expected: expected, ignoreMismatch: ignoreMismatch}
 }
 
 // handleTask processes single download task
 func (downloader *downloaderImpl) handleTask(task *downloadTask) {
-	fmt.Printf("Downloading %s...\n", task.url)
+	downloader.progress.Printf("Downloading %s...\n", task.url)
 
 	resp, err := http.Get(task.url)
 	if err != nil {
@@ -122,13 +132,46 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	}
 	defer outfile.Close()
 
-	_, err = io.Copy(outfile, resp.Body)
+	checksummer := utils.NewChecksumWriter()
+	writers := []io.Writer{outfile, downloader.progress}
+
+	if task.expected.Size != -1 {
+		writers = append(writers, checksummer)
+	}
+
+	w := io.MultiWriter(writers...)
+
+	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
 		task.result <- err
 		return
 	}
 
+	if task.expected.Size != -1 {
+		actual := checksummer.Sum()
+
+		if actual.Size != task.expected.Size {
+			err = fmt.Errorf("%s: size check mismatch %d != %d", task.url, actual.Size, task.expected.Size)
+		} else if task.expected.MD5 != "" && actual.MD5 != task.expected.MD5 {
+			err = fmt.Errorf("%s: md5 hash mismatch %#v != %#v", task.url, actual.MD5, task.expected.MD5)
+		} else if task.expected.SHA1 != "" && actual.SHA1 != task.expected.SHA1 {
+			err = fmt.Errorf("%s: sha1 hash mismatch %#v != %#v", task.url, actual.SHA1, task.expected.SHA1)
+		} else if task.expected.SHA256 != "" && actual.SHA256 != task.expected.SHA256 {
+			err = fmt.Errorf("%s: sha256 hash mismatch %#v != %#v", task.url, actual.SHA256, task.expected.SHA256)
+		}
+
+		if err != nil {
+			if task.ignoreMismatch {
+				downloader.progress.Printf("WARNING: %s\n", err.Error())
+			} else {
+				os.Remove(temppath)
+				task.result <- err
+				return
+			}
+		}
+	}
+
 	err = os.Rename(temppath, task.destination)
 	if err != nil {
 		os.Remove(temppath)
@@ -157,7 +200,14 @@ func (downloader *downloaderImpl) process() {
 // DownloadTemp starts new download to temporary file and returns File
 //
 // Temporary file would be already removed, so no need to cleanup
-func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
+func DownloadTemp(downloader aptly.Downloader, url string) (*os.File, error) {
+	return DownloadTempWithChecksum(downloader, url, utils.ChecksumInfo{Size: -1}, false)
+}
+
+// DownloadTempWithChecksum is a DownloadTemp with checksum verification
+//
+// Temporary file would be already removed, so no need to cleanup
+func DownloadTempWithChecksum(downloader aptly.Downloader, url string, expected utils.ChecksumInfo, ignoreMismatch bool) (*os.File, error) {
 	tempdir, err := ioutil.TempDir(os.TempDir(), "aptly")
 	if err != nil {
 		return nil, err
@@ -166,14 +216,22 @@ func DownloadTemp(downloader Downloader, url string) (*os.File, error) {
 
 	tempfile := filepath.Join(tempdir, "buffer")
 
+	if expected.Size != -1 && downloader.GetProgress() != nil {
+		downloader.GetProgress().InitBar(expected.Size, true)
+	}
+
 	ch := make(chan error, 1)
-	downloader.Download(url, tempfile, ch)
+	downloader.DownloadWithChecksum(url, tempfile, ch, expected, ignoreMismatch)
 
 	err = <-ch
 	if err != nil {
 		return nil, err
 	}
 
+	if expected.Size != -1 && downloader.GetProgress() != nil {
+		downloader.GetProgress().ShutdownBar()
+	}
+
 	file, err := os.Open(tempfile)
 	if err != nil {
 		return nil, err
@@ -203,13 +261,27 @@ var compressionMethods = []struct {
 
 // DownloadTryCompression tries to download from URL .bz2, .gz and raw extension until
 // it finds existing file.
-func DownloadTryCompression(downloader Downloader, url string) (io.Reader, *os.File, error) {
+func DownloadTryCompression(downloader aptly.Downloader, url string, expectedChecksums map[string]utils.ChecksumInfo, ignoreMismatch bool) (io.Reader, *os.File, error) {
 	var err error
 
 	for _, method := range compressionMethods {
 		var file *os.File
 
-		file, err = DownloadTemp(downloader, url+method.extenstion)
+		tryURL := url + method.extenstion
+		foundChecksum := false
+
+		for suffix, expected := range expectedChecksums {
+			if strings.HasSuffix(tryURL, suffix) {
+				file, err = DownloadTempWithChecksum(downloader, tryURL, expected, ignoreMismatch)
+				foundChecksum = true
+				break
+			}
+		}
+
+		if !foundChecksum {
+			file, err = DownloadTemp(downloader, tryURL)
+		}
+
 		if err != nil {
 			continue
 		}

http/download_test.go

@@ -0,0 +1,286 @@
+package http
+
+import (
+	"errors"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/console"
+	"github.com/smira/aptly/utils"
+	"io"
+	"io/ioutil"
+	. "launchpad.net/gocheck"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"time"
+)
+
+type DownloaderSuite struct {
+	tempfile *os.File
+	l        net.Listener
+	url      string
+	ch       chan bool
+	progress aptly.Progress
+}
+
+var _ = Suite(&DownloaderSuite{})
+
+func (s *DownloaderSuite) SetUpTest(c *C) {
+	s.tempfile, _ = ioutil.TempFile(os.TempDir(), "aptly-test")
+	s.l, _ = net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)})
+	s.url = fmt.Sprintf("http://localhost:%d", s.l.Addr().(*net.TCPAddr).Port)
+
+	mux := http.NewServeMux()
+	mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello, %s", r.URL.Path)
+	})
+
+	s.ch = make(chan bool)
+
+	go func() {
+		http.Serve(s.l, mux)
+		s.ch <- true
+	}()
+
+	s.progress = console.NewProgress()
+	s.progress.Start()
+}
+
+func (s *DownloaderSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
+
+	s.l.Close()
+	<-s.ch
+
+	os.Remove(s.tempfile.Name())
+	s.tempfile.Close()
+}
+
+func (s *DownloaderSuite) TestStartupShutdown(c *C) {
+	goroutines := runtime.NumGoroutine()
+
+	d := NewDownloader(10, s.progress)
+	d.Shutdown()
+
+	// wait for goroutines to shutdown
+	time.Sleep(100 * time.Millisecond)
+
+	if runtime.NumGoroutine()-goroutines > 1 {
+		c.Errorf("Number of goroutines %d, expected %d", runtime.NumGoroutine(), goroutines)
+	}
+}
+
+func (s *DownloaderSuite) TestPauseResume(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+
+	d.Pause()
+	d.Resume()
+}
+
+func (s *DownloaderSuite) TestDownloadOK(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+	ch := make(chan error)
+
+	d.Download(s.url+"/test", s.tempfile.Name(), ch)
+	res := <-ch
+	c.Assert(res, IsNil)
+}
+
+func (s *DownloaderSuite) TestDownloadWithChecksum(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+	ch := make(chan error)
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{}, false)
+	res := <-ch
+	c.Assert(res, ErrorMatches, ".*size check mismatch 12 != 0")
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, false)
+	res = <-ch
+	c.Assert(res, ErrorMatches, ".*md5 hash mismatch \"a1acb0fe91c7db45ec4d775192ec5738\" != \"abcdef\"")
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "abcdef"}, true)
+	res = <-ch
+	c.Assert(res, IsNil)
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738"}, false)
+	res = <-ch
+	c.Assert(res, IsNil)
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738", SHA1: "abcdef"}, false)
+	res = <-ch
+	c.Assert(res, ErrorMatches, ".*sha1 hash mismatch \"921893bae6ad6fd818401875d6779254ef0ff0ec\" != \"abcdef\"")
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec"}, false)
+	res = <-ch
+	c.Assert(res, IsNil)
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "abcdef"}, false)
+	res = <-ch
+	c.Assert(res, ErrorMatches, ".*sha256 hash mismatch \"b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac\" != \"abcdef\"")
+
+	d.DownloadWithChecksum(s.url+"/test", s.tempfile.Name(), ch, utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false)
+	res = <-ch
+	c.Assert(res, IsNil)
+}
+
+func (s *DownloaderSuite) TestDownload404(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+	ch := make(chan error)
+
+	d.Download(s.url+"/doesntexist", s.tempfile.Name(), ch)
+	res := <-ch
+	c.Assert(res, ErrorMatches, "HTTP code 404.*")
+}
+
+func (s *DownloaderSuite) TestDownloadConnectError(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+	ch := make(chan error)
+
+	d.Download("http://nosuch.localhost/", s.tempfile.Name(), ch)
+	res := <-ch
+	c.Assert(res, ErrorMatches, ".*no such host")
+}
+
+func (s *DownloaderSuite) TestDownloadFileError(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+	ch := make(chan error)
+
+	d.Download(s.url+"/test", "/", ch)
+	res := <-ch
+	c.Assert(res, ErrorMatches, ".*permission denied")
+}
+
+func (s *DownloaderSuite) TestDownloadTemp(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+
+	f, err := DownloadTemp(d, s.url+"/test")
+	c.Assert(err, IsNil)
+	defer f.Close()
+
+	buf := make([]byte, 1)
+
+	f.Read(buf)
+	c.Assert(buf, DeepEquals, []byte("H"))
+
+	_, err = os.Stat(f.Name())
+	c.Assert(os.IsNotExist(err), Equals, true)
+}
+
+func (s *DownloaderSuite) TestDownloadTempWithChecksum(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+
+	f, err := DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 12, MD5: "a1acb0fe91c7db45ec4d775192ec5738",
+		SHA1: "921893bae6ad6fd818401875d6779254ef0ff0ec", SHA256: "b3c92ee1246176ed35f6e8463cd49074f29442f5bbffc3f8591cde1dcc849dac"}, false)
+	defer f.Close()
+	c.Assert(err, IsNil)
+
+	_, err = DownloadTempWithChecksum(d, s.url+"/test", utils.ChecksumInfo{Size: 13}, false)
+	c.Assert(err, ErrorMatches, ".*size check mismatch 12 != 13")
+}
+
+func (s *DownloaderSuite) TestDownloadTempError(c *C) {
+	d := NewDownloader(2, s.progress)
+	defer d.Shutdown()
+
+	f, err := DownloadTemp(d, s.url+"/doesntexist")
+	c.Assert(err, NotNil)
+	c.Assert(f, IsNil)
+	c.Assert(err, ErrorMatches, "HTTP code 404.*")
+}
+
+const (
+	bzipData = "BZh91AY&SY\xcc\xc3q\xd4\x00\x00\x02A\x80\x00\x10\x02\x00\x0c\x00 \x00!\x9ah3M\x19\x97\x8b\xb9\"\x9c(Hfa\xb8\xea\x00"
+	gzipData = "\x1f\x8b\x08\x00\xc8j\xb0R\x00\x03+I-.\xe1\x02\x00\xc65\xb9;\x05\x00\x00\x00"
+	rawData  = "test"
+)
+
+func (s *DownloaderSuite) TestDownloadTryCompression(c *C) {
+	var buf []byte
+
+	expectedChecksums := map[string]utils.ChecksumInfo{
+		"file.bz2": utils.ChecksumInfo{Size: int64(len(bzipData))},
+		"file.gz":  utils.ChecksumInfo{Size: int64(len(gzipData))},
+		"file":     utils.ChecksumInfo{Size: int64(len(rawData))},
+	}
+
+	// bzip2 only available
+	buf = make([]byte, 4)
+	d := NewFakeDownloader()
+	d.ExpectResponse("http://example.com/file.bz2", bzipData)
+	r, file, err := DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 not available, but gz is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectResponse("http://example.com/file.gz", gzipData)
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// bzip2 & gzip not available, but raw is
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectResponse("http://example.com/file", rawData)
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", expectedChecksums, false)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, rawData)
+	c.Assert(d.Empty(), Equals, true)
+
+	// gzip available, but broken
+	buf = make([]byte, 4)
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectResponse("http://example.com/file.gz", "x")
+	d.ExpectResponse("http://example.com/file", "recovered")
+	r, file, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	c.Assert(err, IsNil)
+	defer file.Close()
+	io.ReadFull(r, buf)
+	c.Assert(string(buf), Equals, "reco")
+	c.Assert(d.Empty(), Equals, true)
+}
+
+func (s *DownloaderSuite) TestDownloadTryCompressionErrors(c *C) {
+	d := NewFakeDownloader()
+	_, _, err := DownloadTryCompression(d, "http://example.com/file", nil, false)
+	c.Assert(err, ErrorMatches, "unexpected request.*")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectError("http://example.com/file", errors.New("403"))
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", nil, false)
+	c.Assert(err, ErrorMatches, "403")
+
+	d = NewFakeDownloader()
+	d.ExpectError("http://example.com/file.bz2", errors.New("404"))
+	d.ExpectError("http://example.com/file.gz", errors.New("404"))
+	d.ExpectResponse("http://example.com/file", rawData)
+	_, _, err = DownloadTryCompression(d, "http://example.com/file", map[string]utils.ChecksumInfo{"file": utils.ChecksumInfo{Size: 7}}, false)
+	c.Assert(err, ErrorMatches, "checksums don't match.*")
+}

http/fake.go

@@ -0,0 +1,137 @@
+package http
+
+import (
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+type expectedRequest struct {
+	URL      string
+	Err      error
+	Response string
+}
+
+// FakeDownloader is like Downloader, but it used in tests
+// to stub out results
+type FakeDownloader struct {
+	expected    []expectedRequest
+	anyExpected map[string]expectedRequest
+}
+
+// Check interface
+var (
+	_ aptly.Downloader = (*FakeDownloader)(nil)
+)
+
+// NewFakeDownloader creates new expected downloader
+func NewFakeDownloader() *FakeDownloader {
+	result := &FakeDownloader{}
+	result.expected = make([]expectedRequest, 0)
+	result.anyExpected = make(map[string]expectedRequest)
+	return result
+}
+
+// ExpectResponse installs expectation on upcoming download with response
+func (f *FakeDownloader) ExpectResponse(url string, response string) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{URL: url, Response: response})
+	return f
+}
+
+// AnyExpectResponse installs expectation on upcoming download with response in any order (url should be unique)
+func (f *FakeDownloader) AnyExpectResponse(url string, response string) *FakeDownloader {
+	f.anyExpected[url] = expectedRequest{URL: url, Response: response}
+	return f
+}
+
+// ExpectError installs expectation on upcoming download with error
+func (f *FakeDownloader) ExpectError(url string, err error) *FakeDownloader {
+	f.expected = append(f.expected, expectedRequest{URL: url, Err: err})
+	return f
+}
+
+// Empty verifies that are planned downloads have happened
+func (f *FakeDownloader) Empty() bool {
+	return len(f.expected) == 0
+}
+
+// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification
+func (f *FakeDownloader) DownloadWithChecksum(url string, filename string, result chan<- error, expected utils.ChecksumInfo, ignoreMismatch bool) {
+	var expectation expectedRequest
+	if len(f.expected) > 0 && f.expected[0].URL == url {
+		expectation, f.expected = f.expected[0], f.expected[1:]
+	} else if _, ok := f.anyExpected[url]; ok {
+		expectation = f.anyExpected[url]
+		delete(f.anyExpected, url)
+	} else {
+		result <- fmt.Errorf("unexpected request for %s", url)
+		return
+	}
+
+	if expectation.Err != nil {
+		result <- expectation.Err
+		return
+	}
+
+	err := os.MkdirAll(filepath.Dir(filename), 0755)
+	if err != nil {
+		result <- err
+		return
+	}
+
+	outfile, err := os.Create(filename)
+	if err != nil {
+		result <- err
+		return
+	}
+	defer outfile.Close()
+
+	cks := utils.NewChecksumWriter()
+	w := io.MultiWriter(outfile, cks)
+
+	_, err = w.Write([]byte(expectation.Response))
+	if err != nil {
+		result <- err
+		return
+	}
+
+	if expected.Size != -1 {
+		if expected.Size != cks.Sum().Size || expected.MD5 != "" && expected.MD5 != cks.Sum().MD5 ||
+			expected.SHA1 != "" && expected.SHA1 != cks.Sum().SHA1 || expected.SHA256 != "" && expected.SHA256 != cks.Sum().SHA256 {
+			if ignoreMismatch {
+				fmt.Printf("WARNING: checksums don't match: %#v != %#v for %s\n", expected, cks.Sum(), url)
+			} else {
+				result <- fmt.Errorf("checksums don't match: %#v != %#v for %s", expected, cks.Sum(), url)
+				return
+			}
+		}
+	}
+
+	result <- nil
+	return
+}
+
+// Download performs fake download by matching against first expectation in the queue
+func (f *FakeDownloader) Download(url string, filename string, result chan<- error) {
+	f.DownloadWithChecksum(url, filename, result, utils.ChecksumInfo{Size: -1}, false)
+}
+
+// Shutdown does nothing
+func (f *FakeDownloader) Shutdown() {
+}
+
+// Pause does nothing
+func (f *FakeDownloader) Pause() {
+}
+
+// Resume does nothing
+func (f *FakeDownloader) Resume() {
+}
+
+// GetProgress returns Progress object
+func (f *FakeDownloader) GetProgress() aptly.Progress {
+	return nil
+}

http/http.go

@@ -0,0 +1,2 @@
+// Package http provides all HTTP-related operations
+package http

http/http_test.go

@@ -0,0 +1,11 @@
+package http
+
+import (
+	. "launchpad.net/gocheck"
+	"testing"
+)
+
+// Launch gocheck tests
+func Test(t *testing.T) {
+	TestingT(t)
+}

main.go

@@ -3,112 +3,93 @@ package main
 import (
 	"fmt"
 	"github.com/gonuts/commander"
-	"github.com/gonuts/flag"
-	"github.com/smira/aptly/database"
-	"github.com/smira/aptly/debian"
+	"github.com/smira/aptly/cmd"
 	"github.com/smira/aptly/utils"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
-// aptly version
-const Version = "0.2"
-
-var cmd *commander.Command
-
-func init() {
-	cmd = &commander.Command{
-		UsageLine: os.Args[0],
-		Short:     "Debian repository management tool",
-		Long: `
-aptly is a tool to create partial and full mirrors of remote
-repositories, filter them, merge, upgrade individual packages,
-take snapshots and publish them back as Debian repositories.`,
-		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
-		Subcommands: []*commander.Command{
-			makeCmdMirror(),
-			makeCmdSnapshot(),
-			makeCmdPublish(),
-			makeCmdVersion(),
-		},
-	}
-
-	cmd.Flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
-	cmd.Flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
-	cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if depdency is 'a|b'")
-	cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
-}
-
-var context struct {
-	downloader        utils.Downloader
-	database          database.Storage
-	packageRepository *debian.Repository
-	dependencyOptions int
-	architecturesList []string
-}
+var (
+	returnCode   = 0
+	errorMessage string
+)
 
 func fatal(err error) {
-	fmt.Printf("ERROR: %s\n", err)
-	os.Exit(1)
+	errorMessage = fmt.Sprintf("ERROR: %s\n", err)
+	returnCode = 1
+}
+
+func loadConfig(command *commander.Command) error {
+	var err error
+
+	configLocation := command.Flag.Lookup("config").Value.String()
+	if configLocation != "" {
+		err = utils.LoadConfig(configLocation, &utils.Config)
+
+		if err != nil {
+			return err
+		}
+	} else {
+		configLocations := []string{
+			filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
+			"/etc/aptly.conf",
+		}
+
+		for _, configLocation := range configLocations {
+			err = utils.LoadConfig(configLocation, &utils.Config)
+			if err == nil {
+				break
+			}
+			if !os.IsNotExist(err) {
+				fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
+				return nil
+			}
+		}
+
+		if err != nil {
+			fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
+			utils.SaveConfig(configLocations[0], &utils.Config)
+		}
+	}
+
+	return nil
 }
 
 func main() {
-	err := cmd.Flag.Parse(os.Args[1:])
+	defer func() {
+		if errorMessage != "" {
+			fmt.Print(errorMessage)
+		}
+		os.Exit(returnCode)
+	}()
+
+	command := cmd.RootCommand()
+
+	err := command.Flag.Parse(os.Args[1:])
 	if err != nil {
 		fatal(err)
+		return
 	}
 
-	configLocations := []string{
-		filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
-		"/etc/aptly.conf",
-	}
-
-	for _, configLocation := range configLocations {
-		err = utils.LoadConfig(configLocation, &utils.Config)
-		if err == nil {
-			break
-		}
-		if !os.IsNotExist(err) {
-			fatal(fmt.Errorf("error loading config file %s: %s", configLocation, err))
-		}
-	}
-
-	if err != nil {
-		fmt.Printf("Config file not found, creating default config at %s\n\n", configLocations[0])
-		utils.SaveConfig(configLocations[0], &utils.Config)
-	}
-
-	context.dependencyOptions = 0
-	if utils.Config.DepFollowSuggests || cmd.Flag.Lookup("dep-follow-suggests").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowSuggests
-	}
-	if utils.Config.DepFollowRecommends || cmd.Flag.Lookup("dep-follow-recommends").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowRecommends
-	}
-	if utils.Config.DepFollowAllVariants || cmd.Flag.Lookup("dep-follow-all-variants").Value.Get().(bool) {
-		context.dependencyOptions |= debian.DepFollowAllVariants
-	}
-
-	context.architecturesList = utils.Config.Architectures
-	optionArchitectures := cmd.Flag.Lookup("architectures").Value.String()
-	if optionArchitectures != "" {
-		context.architecturesList = strings.Split(optionArchitectures, ",")
-	}
-
-	context.downloader = utils.NewDownloader(utils.Config.DownloadConcurrency)
-	defer context.downloader.Shutdown()
-
-	context.database, err = database.OpenDB(filepath.Join(utils.Config.RootDir, "db"))
-	if err != nil {
-		fatal(fmt.Errorf("can't open database: %s", err))
-	}
-	defer context.database.Close()
-
-	context.packageRepository = debian.NewRepository(utils.Config.RootDir)
-
-	err = cmd.Dispatch(cmd.Flag.Args())
+	err = loadConfig(command)
 	if err != nil {
 		fatal(err)
+		return
+	}
+	if returnCode != 0 {
+		return
+	}
+
+	err = cmd.InitContext(command)
+	if err != nil {
+		fatal(err)
+		return
+	}
+	defer cmd.ShutdownContext()
+
+	err = command.Dispatch(command.Flag.Args())
+	if err != nil {
+		fatal(err)
+		return
 	}
 }

man/aptly.1

@@ -0,0 +1,778 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "APTLY" "1" "March 2014" "" ""
+.
+.SH "NAME"
+\fBaptly\fR \- Debian repository management tool
+.
+.SH "SYNOPSIS"
+Common command format:
+.
+.P
+\fBaptly\fR [\fIglobal options\fR\.\.\.] \fIcommand\fR \fIsubcommand\fR [\fIoptions\fR\.\.\.] \fIarguments\fR
+.
+.P
+aptly has integrated help that matches contents of this manual page, to get help, prepend \fBhelp\fR to command name:
+.
+.P
+\fBaptly\fR \fBhelp\fR \fBmirror\fR \fBcreate\fR
+.
+.SH "DESCRIPTION"
+aptly is a tool to create partial and full mirrors of remote repositories, manage local repositories, filter them, merge, upgrade individual packages, take snapshots and publish them back as Debian repositories\.
+.
+.P
+aptly goal is to establish repeatiblity and controlled changes in package environment\. aptly allows to fix set of packages in repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\.
+.
+.SH "CONFIGURATION"
+aptly looks for configuration file in \fB/etc/aptly\.conf\fR and \fB~/\.aptly\.conf\fR, if no config file found, new one is created\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\.
+.
+.P
+Configuration file is stored in JSON format (default values shown below):
+.
+.IP "" 4
+.
+.nf
+
+{
+  "rootDir": "$HOME/\.aptly",
+  "downloadConcurrency": 4,
+  "architectures": [],
+  "dependencyFollowSuggests": false,
+  "dependencyFollowRecommends": false
+  "dependencyFollowAllVariants": false,
+  "dependencyFollowSource": false,
+  "gpgDisableSign": false,
+  "gpgDisableVerify": false,
+  "downloadSourcePackages": false,
+  "ppaDistributorID": "ubuntu",
+  "ppaCodename": ""
+}
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\fBrootDir\fR
+is root of directory storage to store database (\fBrootDir\fR/db), downloaded packages (\fBrootDir\fR/pool) and published repositories (\fBrootDir\fR/public)
+.
+.TP
+\fBdownloadConcurrency\fR
+is a number of parallel download threads to use when downloading packages
+.
+.TP
+\fBarchitectures\fR
+is a list of architectures to process; if left empty defaults to all available architectures; could be overridden with option \fB\-architectures\fR
+.
+.TP
+\fBdependencyFollowSuggests\fR
+follow contents of \fBSuggests:\fR field when processing dependencies for the package
+.
+.TP
+\fBdependencyFollowRecommends\fR
+follow contents of \fBRecommends:\fR field when processing dependencies for the package
+.
+.TP
+\fBdependencyFollowAllVariants\fR
+when dependency looks like \fBpackage\-a | package\-b\fR, follow both variants always
+.
+.TP
+\fBdependencyFollowSource\fR
+follow dependency from binary package to source package
+.
+.TP
+\fBgpgDisableSign\fR
+don\'t sign published repositories with gpg(1), also can be disabled on per\-repo basis using \fB\-skip\-signing\fR flag when publishing
+.
+.TP
+\fBgpgDisableVerify\fR
+don\'t verify remote mirrors with gpg(1), also can be disabled on per\-mirror basis using \fB\-ignore\-signatures\fR flag when creating and updating mirrors
+.
+.TP
+\fBdownloadSourcePackages\fR
+if enabled, all mirrors created would have flag set to download source packages; this setting could be controlled on per\-mirror basis with \fB\-with\-sources\fR flag
+.
+.TP
+\fBppaDistributorID\fR, \fBppaCodename\fR
+specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
+.
+.SH "PACKAGE SPEC"
+Some commands accept package specs to identify list of packages to process\. Package spec is a list of following search conditions:
+.
+.TP
+direct package reference
+reference to exaclty one package\. Format is identical to the way aptly lists packages in show commands with \fB\-with\-packages\fR flag: \fBname_version_arch\fR, e\.g\.: \fBlibmysqlclient18_5\.5\.35\-rel33\.0\-611\.squeeze_amd64\fR
+.
+.TP
+dependency condition
+syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit\.
+.
+.P
+Examples:
+.
+.TP
+\fBmysql\-client\fR
+matches package mysql\-client of any version and architecture (including source)
+.
+.TP
+\fBmysql\-client (>= 3\.6)\fR
+matches package mysql\-client with version greater or equal to 3\.6\. Valid operators for version are: \fB>=\fR, \fB<=\fR, \fB=\fR, \fB>>\fR (strictly greater), \fB<<\fR (strictly less)\.
+.
+.TP
+\fBmysql\-client {i386}\fR
+matches package \fBmysql\-client\fR on architecture \fBi386\fR, architecture \fBall\fR matches all architectures but source\.
+.
+.TP
+\fBmysql\-client (>= 3\.6) {i386}\fR
+version and architecture conditions combined\.
+.
+.P
+When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+.
+.P
+\fBaptly repo import percona stable \'mysql\-client (>= 3\.6)\'\fR
+.
+.SH "GLOBAL OPTIONS"
+.
+.TP
+\-\fBarchitectures\fR=
+list of architectures to consider during (comma\-separated), default to all available
+.
+.TP
+\-\fBconfig\fR=
+location of configuration file (default locations are /etc/aptly\.conf, ~/\.aptly\.conf)
+.
+.TP
+\-\fBcpuprofile\fR=
+write cpu profile to file
+.
+.TP
+\-\fBdep\-follow\-all\-variants\fR=false
+when processing dependencies, follow a & b if depdency is \'a|b\'
+.
+.TP
+\-\fBdep\-follow\-recommends\fR=false
+when processing dependencies, follow Recommends
+.
+.TP
+\-\fBdep\-follow\-source\fR=false
+when processing dependencies, follow from binary to Source packages
+.
+.TP
+\-\fBdep\-follow\-suggests\fR=false
+when processing dependencies, follow Suggests
+.
+.TP
+\-\fBmeminterval\fR=100ms
+memory stats dump interval
+.
+.TP
+\-\fBmemprofile\fR=
+write memory profile to this file
+.
+.TP
+\-\fBmemstats\fR=
+write memory stats periodically to this file
+.
+.SH "CREATE NEW MIRROR"
+\fBaptly\fR \fBmirror\fR \fBcreate\fR \fIname\fR \fIarchive url\fR \fIdistribution\fR [\fIcomponent1\fR \.\.\.]
+.
+.P
+Creates mirror \fIname\fR of remote repository, aptly supports both regular and flat Debian repositories exported via HTTP\. aptly would try download Release file from remote repository and verify its signature\.
+.
+.P
+PPA urls could specified in short format:
+.
+.P
+$ aptly mirror create \fIname\fR ppa:\fIuser\fR/\fIproject\fR
+.
+.P
+Example:
+.
+.P
+$ aptly mirror create wheezy\-main http://mirror\.yandex\.ru/debian/ wheezy main
+.
+.P
+Options:
+.
+.TP
+\-\fBignore\-signatures\fR=false
+disable verification of Release file signatures
+.
+.TP
+\-\fBkeyring\fR=
+gpg keyring to use when verifying Release file (could be specified multiple times)
+.
+.TP
+\-\fBwith\-sources\fR=false
+download source packages in addition to binary packages
+.
+.SH "LIST MIRRORS"
+\fBaptly\fR \fBmirror\fR \fBlist\fR
+.
+.P
+List shows full list of remote repository mirrors\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror list
+.
+.SH "SHOW DETAILS ABOUT MIRROR"
+\fBaptly\fR \fBmirror\fR \fBshow\fR \fIname\fR
+.
+.P
+Shows detailed information about mirror\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror show wheezy\-main
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-packages\fR=false
+show detailed list of packages and versions stored in the mirror
+.
+.SH "DELETE MIRROR"
+\fBaptly\fR \fBmirror\fR \fBdrop\fR \fIname\fR
+.
+.P
+Drop deletes information about remote repository mirror \fIname\fR\. Package data is not deleted (it could be still used by other mirrors or snapshots)\. If mirror is used as source to create a snapshot, aptly would refuse to delete such mirror, use flag \-force to override\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror drop wheezy\-main
+.
+.P
+Options:
+.
+.TP
+\-\fBforce\fR=false
+force mirror deletion even if used by snapshots
+.
+.SH "UPDATE MIRROR"
+\fBaptly\fR \fBmirror\fR \fBupdate\fR \fIname\fR
+.
+.P
+Updates remote mirror (downloads package files and meta information)\. When mirror is created, this command should be run for the first time to fetch mirror contents\. This command could be run many times to get updated repository contents\. If interrupted, command could be restarted safely\.
+.
+.P
+Example:
+.
+.P
+$ aptly mirror update wheezy\-main
+.
+.P
+Options:
+.
+.TP
+\-\fBignore\-checksums\fR=false
+ignore checksum mismatches while downloading package files and metadata
+.
+.TP
+\-\fBignore\-signatures\fR=false
+disable verification of Release file signatures
+.
+.TP
+\-\fBkeyring\fR=
+gpg keyring to use when verifying Release file (could be specified multiple times)
+.
+.SH "ADD PACKAGES TO LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR
+.
+.P
+Command adds packages to local repository from \.deb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would be created and added to database\. Files would be imported to internal package pool\. For source packages, all required files are added as well automatically\. Extra files for source package should be in the same directory as *\.dsc file\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo add testing myapp\-0\.1\.2\.deb incoming/
+.
+.P
+Options:
+.
+.TP
+\-\fBremove\-files\fR=false
+remove files that have been imported successfully into repository
+.
+.SH "COPY PACKAGES BETWEEN LOCAL REPOSITORIES"
+\fBaptly\fR \fBrepo\fR \fBcopy\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\.\.\.\fR
+.
+.P
+Command copy copies packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo copy testing stable \'myapp (=0\.1\.12)\'
+.
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\'t copy, just show what would be copied
+.
+.TP
+\-\fBwith\-deps\fR=false
+follow dependencies when processing package\-spec
+.
+.SH "CREATE LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBcreate\fR \fIname\fR
+.
+.P
+Create local package repository\. Repository would be empty when created, packages could be added from files, copied or moved from another local repository or imported from the mirror\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo create testing
+.
+.P
+Options:
+.
+.TP
+\-\fBcomment\fR=
+any text that would be used to described local repository
+.
+.SH "DELETE LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBdrop\fR \fIname\fR
+.
+.P
+Drop deletes information about local repo\. Package data is not deleted (it could be still used by other mirrors or snapshots)\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo drop local\-repo
+.
+.P
+Options:
+.
+.TP
+\-\fBforce\fR=false
+force local repo deletion even if used by snapshots
+.
+.SH "IMPORT PACKAGES FROM MIRROR TO LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBimport\fR \fIsrc\-mirror\fR \fIdst\-repo\fR \fIpackage\-spec\fR \fB\.\.\.\fR
+.
+.P
+Command import looks up packages matching \fIpackage\-spec\fR in mirror \fIsrc\-mirror\fR and copies them to local repo \fIdst\-repo\fR\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo import wheezy\-main testing nginx
+.
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\'t import, just show what would be imported
+.
+.TP
+\-\fBwith\-deps\fR=false
+follow dependencies when processing package\-spec
+.
+.SH "LIST LOCAL REPOSITORIES"
+\fBaptly\fR \fBrepo\fR \fBlist\fR
+.
+.P
+List shows full list of local package repositories\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo list
+.
+.SH "MOVE PACKAGES BETWEEN LOCAL REPOSITORIES"
+\fBaptly\fR \fBrepo\fR \fBmove\fR \fIsrc\-name\fR \fIdst\-name\fR \fIpackage\-spec\fR \fB\.\.\.\fR
+.
+.P
+Command move moves packages matching \fIpackage\-spec\fR from local repo \fIsrc\-name\fR to local repo \fIdst\-name\fR\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo move testing stable \'myapp (=0\.1\.12)\'
+.
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\'t move, just show what would be moved
+.
+.TP
+\-\fBwith\-deps\fR=false
+follow dependencies when processing package\-spec
+.
+.SH "REMOVE PACKAGES FROM LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBremove\fR \fIname\fR \fIpackage\-spec\fR \fB\.\.\.\fR
+.
+.P
+Commands removes packages matching \fIpackage\-spec\fR from local repository \fIname\fR\. If removed packages are not referenced by other repos or snapshots, they can be removed completely (including files) by running \'aptly db cleanup\'\.
+.
+.P
+Example:
+.
+.P
+$ aptly repo remove testing \'myapp (=0\.1\.12)\'
+.
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\'t remove, just show what would be removed
+.
+.SH "SHOW DETAILS ABOUT LOCAL REPOSITORY"
+\fBaptly\fR \fBrepo\fR \fBshow\fR \fIname\fR
+.
+.P
+Show shows full information about local package repository\.
+.
+.P
+ex: $ aptly repo show testing
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-packages\fR=false
+show list of packages
+.
+.SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
+\fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
+.
+.P
+Command create \fIname\fR from mirror makes persistent immutable snapshot of remote repository mirror\. Snapshot could be published or further modified using merge, pull and other aptly features\.
+.
+.P
+Command create \fIname\fR from repo makes persistent immutable snapshot of local repository\. Snapshot could be processed as mirror snapshots, and mixed with snapshots of remote mirrors\.
+.
+.P
+Command create \fIname\fR empty creates empty snapshot that could be used as a basis for snapshot pull operations, for example\. As snapshots are immutable, creating one empty snapshot should be enough\.
+.
+.P
+Example:
+.
+.P
+$ aptly snapshot create wheezy\-main\-today from mirror wheezy\-main
+.
+.SH "LIST SNAPSHOTS"
+\fBaptly\fR \fBsnapshot\fR \fBlist\fR
+.
+.P
+Command list shows full list of snapshots created\.
+.
+.P
+Example:
+.
+.P
+$ aptly snapshot list
+.
+.SH "SHOWS DETAILS ABOUT SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBshow\fR \fIname\fR
+.
+.P
+Command show displays full information about snapshot\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot show wheezy\-main
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-packages\fR=false
+show list of packages
+.
+.SH "VERIFY DEPENDENCIES IN SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBverify\fR \fIname\fR [\fIsource\fR \.\.\.]
+.
+.P
+Verify does depenency resolution in snapshot \fIname\fR, possibly using additional snapshots \fIsource\fR as dependency sources\. All unsatisfied dependencies are printed\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot verify wheezy\-main wheezy\-contrib wheezy\-non\-free
+.
+.fi
+.
+.IP "" 0
+.
+.SH "PULL PACKAGES FROM ANOTHER SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBpull\fR \fIname\fR \fIsource\fR \fIdestination\fR \fIpackage\-name\fR \fB\.\.\.\fR
+.
+.P
+Command pull pulls new packages along with its dependencies to snapshot \fIname\fR from snapshot \fIsource\fR\. Pull can upgrade package version in \fIname\fR with versions from \fIsource\fR following dependencies\. New snapshot \fIdestination\fR is created as result of this process\. Packages could be specified simply as \'package\-name\' or as dependency \'package\-name (>= version)\'\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot pull wheezy\-main wheezy\-backports wheezy\-new\-xorg xorg\-server\-server
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBdry\-run\fR=false
+don\'t create destination snapshot, just show what would be pulled
+.
+.TP
+\-\fBno\-deps\fR=false
+don\'t process dependencies, just pull listed packages
+.
+.TP
+\-\fBno\-remove\fR=false
+don\'t remove other package versions when pulling package
+.
+.SH "DIFFERENCE BETWEEN TWO SNAPSHOTS"
+\fBaptly\fR \fBsnapshot\fR \fBdiff\fR \fIname\-a\fR \fIname\-b\fR
+.
+.P
+Displays difference in packages between two snapshots\. Snapshot is a list of packages, so difference between snapshots is a difference between package lists\. Package could be either completely missing in one snapshot, or package is present in both snapshots with different versions\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot diff \-only\-matching wheezy\-main wheezy\-backports
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBonly\-matching\fR=false
+display diff only for matching packages (don\'t display missing packages)
+.
+.SH "MERGES SNAPSHOTS"
+\fBaptly\fR \fBsnapshot\fR \fBmerge\fR \fIdestination\fR \fIsource\fR [\fIsource\fR\.\.\.]
+.
+.P
+Merge merges several \fIsource\fR snapshots into one \fIdestination\fR snapshot\. Merge happens from left to right\. Packages with the same name\-architecture pair are replaced during merge (package from latest snapshot on the list wins)\. If run with only one source snapshot, merge copies \fIsource\fR into \fIdestination\fR\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot merge wheezy\-w\-backports wheezy\-main wheezy\-backports
+.
+.fi
+.
+.IP "" 0
+.
+.SH "DELETE SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBdrop\fR \fIname\fR
+.
+.P
+Drop removes information about snapshot\. If snapshot is published, it can\'t be dropped\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot drop wheezy\-main
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBforce\fR=false
+remove snapshot even if it was used as source for other snapshots
+.
+.SH "PUBLISH SNAPSHOT"
+\fBaptly\fR \fBpublish\fR \fBsnapshot\fR \fIname\fR [\fIprefix\fR]
+.
+.P
+Command publish publishes snapshot as Debian repository ready to be consumed by apt tools\. Published repostiories appear under rootDir/public directory\. Valid GPG key is required for publishing\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly publish snapshot wheezy\-main
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBcomponent\fR=
+component name to publish
+.
+.TP
+\-\fBdistribution\fR=
+distribution name to publish
+.
+.TP
+\-\fBgpg\-key\fR=
+GPG key ID to use when signing the release
+.
+.TP
+\-\fBkeyring\fR=
+GPG keyring to use (instead of default)
+.
+.TP
+\-\fBsecret\-keyring\fR=
+GPG secret keyring to use (instead of default)
+.
+.TP
+\-\fBskip\-signing\fR=false
+don\'t sign Release files with GPG
+.
+.SH "LIST OF PUBLISHED REPOSITORIES"
+\fBaptly\fR \fBpublish\fR \fBlist\fR
+.
+.P
+Display list of currently published snapshots\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly publish list
+.
+.fi
+.
+.IP "" 0
+.
+.SH "REMOVE PUBLISHED REPOSITORY"
+\fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [\fIprefix\fR]
+.
+.P
+Command removes whatever has been published under specified \fIprefix\fR and \fIdistribution\fR name\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly publish drop wheezy
+.
+.fi
+.
+.IP "" 0
+.
+.SH "CLEANUP DB AND PACKAGE POOL"
+\fBaptly\fR \fBdb\fR \fBcleanup\fR
+.
+.P
+Database cleanup removes information about unreferenced packages and removes files in the package pool that aren\'t used by packages anymore
+.
+.P
+Example:
+.
+.P
+$ aptly db cleanup
+.
+.SH "HTTP SERVE PUBLISHED REPOSITORIES"
+\fBaptly\fR \fBserve\fR
+.
+.P
+Command serve starts embedded HTTP server (not suitable for real production usage) to serve contents of public/ subdirectory of aptly\'s root that contains published repositories\.
+.
+.P
+Example:
+.
+.P
+$ aptly serve \-listen=:8080
+.
+.P
+Options:
+.
+.TP
+\-\fBlisten\fR=:8080
+host:port for HTTP listening
+.
+.SH "RENDER GRAPH OF RELATIONSHIPS"
+\fBaptly\fR \fBgraph\fR
+.
+.P
+Command graph displays relationship between mirrors, local repositories, snapshots and published repositories using graphviz package to render graph as image\.
+.
+.P
+Example:
+.
+.P
+$ aptly graph
+.
+.SH "ENVIRONMENT"
+If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
+.
+.SH "RETURN VALUES"
+\fBaptly\fR exists with 0 on success and with 1 on failure\.
+.
+.SH "AUTHORS"
+Andrey Smirnov (me@smira\.ru)

man/aptly.1.ronn.tmpl

@@ -0,0 +1,176 @@
+{{define "main"}}aptly(1) -- {{.Short}}
+=============================================
+
+## SYNOPSIS
+
+Common command format:
+
+  `aptly` [<global options>...] <command> <subcommand> [<options>...] <arguments>
+
+aptly has integrated help that matches contents of this manual page, to get help, prepend
+`help` to command name:
+
+   `aptly` `help` `mirror` `create`
+
+## DESCRIPTION
+
+{{.Long}}
+
+## CONFIGURATION
+
+aptly looks for configuration file in `/etc/aptly.conf` and `~/.aptly.conf`, if no config file
+found, new one is created. If `-config=` flag is specified, aptly would use config file at specified
+location. Also aptly needs root directory for database, package and published repository storage.
+If not specified, directory defaults to `~/.aptly`, it will be created if missing.
+
+Configuration file is stored in JSON format (default values shown below):
+
+    {
+      "rootDir": "$HOME/.aptly",
+      "downloadConcurrency": 4,
+      "architectures": [],
+      "dependencyFollowSuggests": false,
+      "dependencyFollowRecommends": false
+      "dependencyFollowAllVariants": false,
+      "dependencyFollowSource": false,
+      "gpgDisableSign": false,
+      "gpgDisableVerify": false,
+      "downloadSourcePackages": false,
+      "ppaDistributorID": "ubuntu",
+      "ppaCodename": ""
+    }
+
+Options:
+
+  * `rootDir`:
+    is root of directory storage to store database (`rootDir`/db), downloaded packages (`rootDir`/pool) and
+    published repositories (`rootDir`/public)
+
+  * `downloadConcurrency`:
+    is a number of parallel download threads to use when downloading packages
+
+  * `architectures`:
+    is a list of architectures to process; if left empty defaults to all available architectures; could be
+    overridden with option `-architectures`
+
+  * `dependencyFollowSuggests`:
+    follow contents of `Suggests:` field when processing dependencies for the package
+
+  * `dependencyFollowRecommends`:
+    follow contents of `Recommends:` field when processing dependencies for the package
+
+  * `dependencyFollowAllVariants`:
+    when dependency looks like `package-a | package-b`, follow both variants always
+
+  * `dependencyFollowSource`:
+    follow dependency from binary package to source package
+
+  * `gpgDisableSign`:
+    don't sign published repositories with gpg(1), also can be disabled on
+    per-repo basis using `-skip-signing` flag when publishing
+
+  * `gpgDisableVerify`:
+    don't verify remote mirrors with gpg(1), also can be disabled on
+    per-mirror basis using `-ignore-signatures` flag when creating and updating mirrors
+
+  * `downloadSourcePackages`:
+    if enabled, all mirrors created would have flag set to download source packages;
+    this setting could be controlled on per-mirror basis with `-with-sources` flag
+
+  * `ppaDistributorID`, `ppaCodename`:
+    specifies paramaters for short PPA url expansion, if left blank they default
+    to output of `lsb_release` command
+
+## PACKAGE SPEC
+
+Some commands accept package specs to identify list of packages to process.
+Package spec is a list of following search conditions:
+
+  * direct package reference:
+    reference to exaclty one package. Format is identical to the way aptly lists packages in
+    show commands with `-with-packages` flag: `name_version_arch`,
+    e.g.: `libmysqlclient18_5.5.35-rel33.0-611.squeeze_amd64`
+
+  * dependency condition:
+    syntax follows Debian dependency specification: package_name followed by optional version specification and architecture limit.
+
+Examples:
+
+  * `mysql-client`:
+     matches package mysql-client of any version and architecture (including source)
+
+  * `mysql-client (>= 3.6)`:
+     matches package mysql-client with version greater or equal to 3.6. Valid operators for
+     version are: `>=`, `<=`, `=`, `>>` (strictly greater), `<<` (strictly less).
+
+  * `mysql-client {i386}`:
+     matches package `mysql-client` on architecture `i386`, architecture `all` matches all architectures but source.
+
+  * `mysql-client (>= 3.6) {i386}`:
+      version and architecture conditions combined.
+
+When specified on command line, condition may have to be quoted according to shell rules, so that it stays single argument:
+
+  `aptly repo import percona stable 'mysql-client (>= 3.6)'`
+
+
+## GLOBAL OPTIONS
+
+{{template "options" .}}
+
+{{template "command" findCommand . "mirror"}}
+
+{{template "command" findCommand . "repo"}}
+
+{{template "command" findCommand . "snapshot"}}
+
+{{template "command" findCommand . "publish"}}
+
+{{template "command" findCommand . "db"}}
+
+{{template "command" findCommand . "serve"}}
+
+{{template "command" findCommand . "graph"}}
+
+## ENVIRONMENT
+
+If environment variable `HTTP_PROXY` is set `aptly` would use its value
+to proxy all HTTP requests.
+
+## RETURN VALUES
+
+`aptly` exists with 0 on success and with 1 on failure.
+
+## AUTHORS
+
+Andrey Smirnov (me@smira.ru)
+
+{{end}}
+
+{{/* command list */}}
+{{define "command"}}
+{{if .Runnable}}
+## {{toUpper .Short}}
+
+{{capitalize .Parent.FullSpacedName}} {{capitalize .UsageLine}}
+
+{{.Long}}
+
+{{if (allFlags .Flag | len) gt 0}}
+Options:
+
+{{template "options" .}}
+{{end}}
+
+{{end}}
+
+{{range .Subcommands}}{{template "command" .}}{{end}}
+{{end}}
+
+{{/* options layout */}}
+{{define "options"}}
+{{range allFlags .Flag}}
+  * -`{{.Name}}`={{.DefValue}}:
+    {{.Usage}}
+{{end}}
+{{end}}

man/gen.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"fmt"
+	"github.com/gonuts/commander"
+	"github.com/gonuts/flag"
+	"github.com/smira/aptly/cmd"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"text/template"
+)
+
+func allFlags(flags *flag.FlagSet) []*flag.Flag {
+	result := []*flag.Flag{}
+	flags.VisitAll(func(f *flag.Flag) {
+		result = append(result, f)
+	})
+	return result
+}
+
+func findCommand(cmd *commander.Command, name string) (*commander.Command, error) {
+	for _, c := range cmd.Subcommands {
+		if c.Name() == name {
+			return c, nil
+		}
+	}
+
+	return nil, fmt.Errorf("command %s not found", name)
+}
+
+func capitalize(s string) string {
+	parts := strings.Split(s, " ")
+	for i, part := range parts {
+		if part[0] != '<' && part[0] != '[' && part[len(part)-1] != '>' && part[len(part)-1] != ']' {
+			parts[i] = "`" + part + "`"
+		}
+	}
+
+	return strings.Join(parts, " ")
+}
+
+func main() {
+	command := cmd.RootCommand()
+	command.UsageLine = "aptly"
+	command.Dispatch(nil)
+
+	_, _File, _, _ := runtime.Caller(0)
+	_File, _ = filepath.Abs(_File)
+
+	templ := template.New("man").Funcs(template.FuncMap{
+		"allFlags":    allFlags,
+		"findCommand": findCommand,
+		"toUpper":     strings.ToUpper,
+		"capitalize":  capitalize,
+	})
+	template.Must(templ.ParseFiles(filepath.Join(filepath.Dir(_File), "aptly.1.ronn.tmpl")))
+
+	output, err := os.Create(filepath.Join(filepath.Dir(_File), "aptly.1.ronn"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = templ.ExecuteTemplate(output, "main", command)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	output.Close()
+
+	out, err := exec.Command("ronn", filepath.Join(filepath.Dir(_File), "aptly.1.ronn")).CombinedOutput()
+	if err != nil {
+		os.Stdout.Write(out)
+		log.Fatal(err)
+	}
+
+	cmd := exec.Command("man", filepath.Join(filepath.Dir(_File), "aptly.1"))
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		log.Fatal(err)
+	}
+}

mem.gp

@@ -0,0 +1,6 @@
+set output 'mem.png'
+set term png
+set key box left
+set xlabel "Time (msec)"
+set ylabel "Mem (MB)"
+plot "mem.dat" using 1:($2/1e6) title 'HeapSys' with lines, "mem.dat" using 1:($3/1e6) title 'HeapAlloc' with lines, "mem.dat" using 1:($4/1e6) title 'HeapIdle' with lines