Disable support for aptly task for now. #96

Config option should act as default, while flag should override it only if set.

.travis.yml

@@ -2,13 +2,16 @@ language: go
 
 go:
   - 1.2.1
-  - 1.3
+  - 1.3.1
   - tip
 
 env:
   global:
     - secure: "YSwtFrMqh4oUvdSQTXBXMHHLWeQgyNEL23ChIZwU0nuDGIcQZ65kipu0PzefedtUbK4ieC065YCUi4UDDh6gPotB/Wu1pnYg3dyQ7rFvhaVYAAUEpajAdXZhlx+7+J8a4FZMeC/kqiahxoRgLbthF9019ouIqhGB9zHKI6/yZwc="
-
+    - secure: "V7OjWrfQ8UbktgT036jYQPb/7GJT3Ol9LObDr8FYlzsQ+F1uj2wLac6ePuxcOS4FwWOJinWGM1h+JiFkbxbyFqfRNJ0jj0O2p93QyDojxFVOn1mXqqvV66KFqAWR2Vzkny/gDvj8LTvdB1cgAIm2FNOkQc6E1BFnyWS2sN9ea5E="
+before_install:
+  - sudo apt-get update -qq
+  - sudo apt-get install -y python-boto
 install:
   - make prepare
 

AUTHORS

@@ -3,4 +3,5 @@ List of contributors, in chronological order:
 * Andrey Smirnov (https://github.com/smira)
 * Sebastien Binet (https://github.com/sbinet)
 * Ryan Uber (https://github.com/ryanuber)
-* Simon Aquino (https://github.com/simonaquino)
+* Simon Aquino (https://github.com/simonaquino)
+* Vincent Batoufflet (https://github.com/vbatoufflet)

Gomfile

@@ -4,11 +4,14 @@ gom 'code.google.com/p/gographviz', :commit => '454bc64fdfa2'
 gom 'code.google.com/p/mxk/go1/flowcontrol', :commit => '5ff2502e2556'
 gom 'code.google.com/p/snappy-go/snappy', :commit => '12e4b4183793'
 gom 'github.com/cheggaaa/pb', :commit => '74be7a1388046f374ac36e93d46f5d56e856f827'
-gom 'github.com/mitchellh/goamz/s3', :commit => '55f224c07975fddef9d2116600c664e30df3d594'
+gom 'github.com/vaughan0/go-ini', :commit => 'a98ad7ee00ec53921f08832bc06ecf7fd600e6a1'
+gom 'github.com/mattn/go-shellwords', :commit => 'c7ca6f94add751566a61cf2199e1de78d4c3eee4'
+gom 'github.com/mitchellh/goamz/s3', :commit => 'e7664b32019f31fd1bdf33f9e85f28722f700405'
 gom 'github.com/mkrautz/goar', :commit => '36eb5f3452b1283a211fa35bc00c646fd0db5c4b'
 gom 'github.com/smira/commander', :commit => 'f408b00e68d5d6e21b9f18bd310978dafc604e47'
-gom 'github.com/smira/flag', :commit => '0d0aac2addb39050f45e92c5a6252926096dc841'
-gom 'github.com/syndtr/goleveldb/leveldb', :commit => '9888007'
+gom 'github.com/smira/flag', :commit => '357ed3e599ffcbd4aeaa828e1d10da2df3ea5107'
+gom 'github.com/smira/go-ftp-protocol/protocol', :commit => '066b75c2b70dca7ae10b1b88b47534a3c31ccfaa'
+gom 'github.com/syndtr/goleveldb/leveldb', :commit => 'e2fa4e6ac1cc41a73bc9fd467878ecbf65df5cc3'
 gom 'github.com/ugorji/go/codec', :commit => '71c2886f5a673a35f909803f38ece5810165097b'
 gom 'github.com/wsxiaoys/terminal/color', :commit => '5668e431776a7957528361f90ce828266c69ed08'
 

Makefile

@@ -79,5 +79,6 @@ src-package:
 	rm -rf aptly-$(VERSION)/src/github.com/smira/aptly/_vendor/{pkg,bin}
 	tar cyf aptly-$(VERSION)-src.tar.bz2 aptly-$(VERSION)
 	rm -rf aptly-$(VERSION)
+	curl -T aptly-$(VERSION)-src.tar.bz2 -usmira:$(BINTRAY_KEY) https://api.bintray.com/content/smira/aptly/aptly/$(VERSION)/$(VERSION)/aptly-$(VERSION)-src.tar.bz2
 
 .PHONY: coverage.out

README.rst

@@ -11,9 +11,11 @@ aptly
 .. image:: http://gobuild.io/badge/github.com/smira/aptly/download.png
     :target: http://gobuild.io/github.com/smira/aptly
 
-
 Aptly is a swiss army knife for Debian repository management.
 
+.. image:: http://www.aptly.info/img/aptly_logo.png
+    :target: http://www.aptly.info/
+
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support use
 mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
 
@@ -24,14 +26,14 @@ Aptly features: ("+" means planned features)
 * publish snapshot as Debian repository, ready to be consumed by apt
 * controlled update of one or more packages in snapshot from upstream mirror, tracking dependencies
 * merge two or more snapshots into one
-* filter repository by search query, pulling dependencies when required (+)
-* publish self-made packages as Debian repositories (+)
+* filter repository by search query, pulling dependencies when required
+* publish self-made packages as Debian repositories
 * mirror repositories "as-is" (without resigning with user's key) (+)
 * support for yum repositories (+)
 
 Current limitations:
 
-* debian-installer and translations not supported yet
+* translations are not supported yet
 
 Download
 --------

aptly/interfaces.go

@@ -90,6 +90,8 @@ type Downloader interface {
 	// Shutdown stops downloader after current tasks are finished,
 	// but doesn't process rest of queue
 	Shutdown()
+	// Abort stops downloader without waiting for shutdown
+	Abort()
 	// GetProgress returns Progress object
 	GetProgress() Progress
 }

aptly/version.go

@@ -1,7 +1,7 @@
 package aptly
 
 // Version of aptly
-const Version = "0.7.1"
+const Version = "0.8"
 
 // Enable debugging features?
 const EnableDebug = false

cmd/cmd.go

@@ -34,6 +34,18 @@ func ListPackagesRefList(reflist *deb.PackageRefList) (err error) {
 	return
 }
 
+// LookupOption checks boolean flag with default (usually config) and command-line
+// setting
+func LookupOption(defaultValue bool, flags *flag.FlagSet, name string) (result bool) {
+	result = defaultValue
+
+	if flags.IsSet(name) {
+		result = flags.Lookup(name).Value.Get().(bool)
+	}
+
+	return
+}
+
 // RootCommand creates root command in command tree
 func RootCommand() *commander.Command {
 	cmd := &commander.Command{
@@ -46,9 +58,9 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
@@ -59,8 +71,11 @@ package environment to new version.`,
 			makeCmdRepo(),
 			makeCmdServe(),
 			makeCmdSnapshot(),
+			// Disabled on no docs
+			//makeCmdTask(),
 			makeCmdPublish(),
 			makeCmdVersion(),
+			makeCmdPackage(),
 		},
 	}
 

cmd/context.go

@@ -22,8 +22,8 @@ import (
 
 // AptlyContext is a common context shared by all commands
 type AptlyContext struct {
-	flags        *flag.FlagSet
-	configLoaded bool
+	flags, globalFlags *flag.FlagSet
+	configLoaded       bool
 
 	progress          aptly.Progress
 	downloader        aptly.Downloader
@@ -65,7 +65,7 @@ func (context *AptlyContext) Config() *utils.ConfigStructure {
 	if !context.configLoaded {
 		var err error
 
-		configLocation := context.flags.Lookup("config").Value.String()
+		configLocation := context.globalFlags.Lookup("config").Value.String()
 		if configLocation != "" {
 			err = utils.LoadConfig(configLocation, &utils.Config)
 
@@ -104,16 +104,16 @@ func (context *AptlyContext) Config() *utils.ConfigStructure {
 func (context *AptlyContext) DependencyOptions() int {
 	if context.dependencyOptions == -1 {
 		context.dependencyOptions = 0
-		if context.Config().DepFollowSuggests || context.flags.Lookup("dep-follow-suggests").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowSuggests, context.globalFlags, "dep-follow-suggests") {
 			context.dependencyOptions |= deb.DepFollowSuggests
 		}
-		if context.Config().DepFollowRecommends || context.flags.Lookup("dep-follow-recommends").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowRecommends, context.globalFlags, "dep-follow-recommends") {
 			context.dependencyOptions |= deb.DepFollowRecommends
 		}
-		if context.Config().DepFollowAllVariants || context.flags.Lookup("dep-follow-all-variants").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowAllVariants, context.globalFlags, "dep-follow-all-variants") {
 			context.dependencyOptions |= deb.DepFollowAllVariants
 		}
-		if context.Config().DepFollowSource || context.flags.Lookup("dep-follow-source").Value.Get().(bool) {
+		if LookupOption(context.Config().DepFollowSource, context.globalFlags, "dep-follow-source") {
 			context.dependencyOptions |= deb.DepFollowSource
 		}
 	}
@@ -125,7 +125,7 @@ func (context *AptlyContext) DependencyOptions() int {
 func (context *AptlyContext) ArchitecturesList() []string {
 	if context.architecturesList == nil {
 		context.architecturesList = context.Config().Architectures
-		optionArchitectures := context.flags.Lookup("architectures").Value.String()
+		optionArchitectures := context.globalFlags.Lookup("architectures").Value.String()
 		if optionArchitectures != "" {
 			context.architecturesList = strings.Split(optionArchitectures, ",")
 		}
@@ -181,6 +181,36 @@ func (context *AptlyContext) Database() (database.Storage, error) {
 	return context.database, nil
 }
 
+// CloseDatabase closes the db temporarily
+func (context *AptlyContext) CloseDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	return context.database.Close()
+}
+
+// ReOpenDatabase reopens the db after close
+func (context *AptlyContext) ReOpenDatabase() error {
+	if context.database == nil {
+		return nil
+	}
+
+	const MaxTries = 10
+	const Delay = 10 * time.Second
+
+	for try := 0; try < MaxTries; try++ {
+		err := context.database.ReOpen()
+		if err == nil || strings.Index(err.Error(), "resource temporarily unavailable") == -1 {
+			return err
+		}
+		context.Progress().Printf("Unable to reopen database, sleeping %s\n", Delay)
+		<-time.After(Delay)
+	}
+
+	return fmt.Errorf("unable to reopen the DB, maximum number of retries reached")
+}
+
 // CollectionFactory builds factory producing all kinds of collections
 func (context *AptlyContext) CollectionFactory() *deb.CollectionFactory {
 	if context.collectionFactory == nil {
@@ -203,7 +233,7 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	return context.packagePool
 }
 
-// PublishedStorage returns instance of PublishedStorage
+// GetPublishedStorage returns instance of PublishedStorage
 func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
 	publishedStorage, ok := context.publishedStorages[name]
 	if !ok {
@@ -217,7 +247,8 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 
 			var err error
 			publishedStorage, err = s3.NewPublishedStorage(params.AccessKeyID, params.SecretAccessKey,
-				params.Region, params.Bucket, params.ACL, params.Prefix)
+				params.Region, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
+				params.EncryptionMethod, params.PlusWorkaround)
 			if err != nil {
 				Fatal(err)
 			}
@@ -230,6 +261,11 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 	return publishedStorage
 }
 
+// UpdateFlags sets internal copy of flags in the context
+func (context *AptlyContext) UpdateFlags(flags *flag.FlagSet) {
+	context.flags = flags
+}
+
 // ShutdownContext shuts context down
 func ShutdownContext() {
 	if aptly.EnableDebug {
@@ -250,12 +286,27 @@ func ShutdownContext() {
 	}
 	if context.database != nil {
 		context.database.Close()
+		context.database = nil
 	}
 	if context.downloader != nil {
-		context.downloader.Shutdown()
+		context.downloader.Abort()
+		context.downloader = nil
 	}
 	if context.progress != nil {
 		context.progress.Shutdown()
+		context.progress = nil
+	}
+}
+
+// CleanupContext does partial shutdown of context
+func CleanupContext() {
+	if context.downloader != nil {
+		context.downloader.Shutdown()
+		context.downloader = nil
+	}
+	if context.progress != nil {
+		context.progress.Shutdown()
+		context.progress = nil
 	}
 }
 
@@ -263,8 +314,13 @@ func ShutdownContext() {
 func InitContext(flags *flag.FlagSet) error {
 	var err error
 
+	if context != nil {
+		panic("context already initialized")
+	}
+
 	context = &AptlyContext{
 		flags:             flags,
+		globalFlags:       flags,
 		dependencyOptions: -1,
 		publishedStorages: map[string]aptly.PublishedStorage{},
 	}

cmd/mirror.go

@@ -8,7 +8,7 @@ import (
 )
 
 func getVerifier(flags *flag.FlagSet) (utils.Verifier, error) {
-	if context.Config().GpgDisableVerify || flags.Lookup("ignore-signatures").Value.Get().(bool) {
+	if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
 		return nil, nil
 	}
 
@@ -56,6 +56,7 @@ func makeCmdMirror() *commander.Command {
 			makeCmdMirrorUpdate(),
 			makeCmdMirrorRename(),
 			makeCmdMirrorEdit(),
+			makeCmdMirrorSearch(),
 		},
 	}
 }

cmd/mirror_create.go

@@ -16,7 +16,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	downloadSources := context.Config().DownloadSourcePackages || context.flags.Lookup("with-sources").Value.Get().(bool)
+	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.flags, "with-sources")
+	downloadUdebs := context.flags.Lookup("with-udebs").Value.Get().(bool)
 
 	var (
 		mirrorName, archiveURL, distribution string
@@ -33,7 +34,8 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 		archiveURL, distribution, components = args[1], args[2], args[3:]
 	}
 
-	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), downloadSources)
+	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
+		downloadSources, downloadUdebs)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -74,7 +76,7 @@ func makeCmdMirrorCreate() *commander.Command {
 		Short:     "create new mirror",
 		Long: `
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
 line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
@@ -90,6 +92,7 @@ Example:
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "gpg keyring to use when verifying Release file (could be specified multiple times)")

cmd/mirror_drop.go

@@ -20,6 +20,11 @@ func aptlyMirrorDrop(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to drop: %s", err)
 	}
 
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to drop: %s", err)
+	}
+
 	force := context.flags.Lookup("force").Value.Get().(bool)
 	if !force {
 		snapshots := context.CollectionFactory().SnapshotCollection().ByRemoteRepoSource(repo)

cmd/mirror_edit.go

@@ -19,15 +19,28 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to edit: %s", err)
+	}
+
 	context.flags.Visit(func(flag *flag.Flag) {
 		switch flag.Name {
 		case "filter":
 			repo.Filter = flag.Value.String()
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-sources":
+			repo.DownloadSources = flag.Value.Get().(bool)
+		case "with-udebs":
+			repo.DownloadUdebs = flag.Value.Get().(bool)
 		}
 	})
 
+	if repo.IsFlat() && repo.DownloadUdebs {
+		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -35,6 +48,15 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
+	if context.globalFlags.Lookup("architectures").Value.String() != "" {
+		repo.Architectures = context.ArchitecturesList()
+
+		err = repo.Fetch(context.Downloader(), nil)
+		if err != nil {
+			return fmt.Errorf("unable to edit: %s", err)
+		}
+	}
+
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
@@ -48,10 +70,10 @@ func makeCmdMirrorEdit() *commander.Command {
 	cmd := &commander.Command{
 		Run:       aptlyMirrorEdit,
 		UsageLine: "edit <name>",
-		Short:     "edit properties of mirorr",
+		Short:     "edit mirror settings",
 		Long: `
-Command edit allows to change settings of mirror:
-filters.
+Command edit allows one to change settings of mirror:
+filters, list of architectures.
 
 Example:
 
@@ -62,6 +84,8 @@ Example:
 
 	cmd.Flag.String("filter", "", "filter packages in mirror")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
+	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
+	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
 
 	return cmd
 }

cmd/mirror_rename.go

@@ -24,6 +24,11 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to rename: %s", err)
 	}
 
+	err = repo.CheckLock()
+	if err != nil {
+		return fmt.Errorf("unable to rename: %s", err)
+	}
+
 	_, err = context.CollectionFactory().RemoteRepoCollection().ByName(newName)
 	if err == nil {
 		return fmt.Errorf("unable to rename: mirror %s already exists", newName)

cmd/mirror_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdMirrorSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search mirror for packages matching query",
+		Long: `
+Command search displays list of packages in mirror that match package query
+
+Example:
+
+    $ aptly mirror search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-mirror-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/mirror_show.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"fmt"
+	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -28,6 +29,9 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 	}
 
 	fmt.Printf("Name: %s\n", repo.Name)
+	if repo.Status == deb.MirrorUpdating {
+		fmt.Printf("Status: In Update (PID %d)\n", repo.WorkerPID)
+	}
 	fmt.Printf("Archive Root URL: %s\n", repo.ArchiveRoot)
 	fmt.Printf("Distribution: %s\n", repo.Distribution)
 	fmt.Printf("Components: %s\n", strings.Join(repo.Components, ", "))
@@ -37,6 +41,11 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
 		downloadSources = "yes"
 	}
 	fmt.Printf("Download Sources: %s\n", downloadSources)
+	downloadUdebs := "no"
+	if repo.DownloadUdebs {
+		downloadUdebs = "yes"
+	}
+	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := "no"

cmd/mirror_update.go

@@ -4,8 +4,12 @@ import (
 	"fmt"
 	"github.com/smira/aptly/deb"
 	"github.com/smira/aptly/query"
+	"github.com/smira/aptly/utils"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
+	"os"
+	"os/signal"
+	"strings"
 )
 
 func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
@@ -27,6 +31,14 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	force := context.flags.Lookup("force").Value.Get().(bool)
+	if !force {
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	ignoreMismatch := context.flags.Lookup("ignore-checksums").Value.Get().(bool)
 
 	verifier, err := getVerifier(context.flags)
@@ -39,21 +51,112 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	var filterQuery deb.PackageQuery
-
-	if repo.Filter != "" {
-		filterQuery, err = query.Parse(repo.Filter)
-		if err != nil {
-			return fmt.Errorf("unable to update: %s", err)
-		}
-	}
-
-	err = repo.Download(context.Progress(), context.Downloader(), context.CollectionFactory(), context.PackagePool(), ignoreMismatch,
-		context.DependencyOptions(), filterQuery)
+	context.Progress().Printf("Downloading & parsing package files...\n")
+	err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.Filter != "" {
+		context.Progress().Printf("Applying filter...\n")
+		var filterQuery deb.PackageQuery
+
+		filterQuery, err = query.Parse(repo.Filter)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+
+		var oldLen, newLen int
+		oldLen, newLen, err = repo.ApplyFilter(context.DependencyOptions(), filterQuery)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+		context.Progress().Printf("Packages filtered: %d -> %d.\n", oldLen, newLen)
+	}
+
+	var (
+		downloadSize int64
+		queue        []deb.PackageDownloadTask
+	)
+
+	context.Progress().Printf("Building download queue...\n")
+	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool())
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	defer func() {
+		// on any interruption, unlock the mirror
+		err := context.ReOpenDatabase()
+		if err == nil {
+			repo.MarkAsIdle()
+			context.CollectionFactory().RemoteRepoCollection().Update(repo)
+		}
+	}()
+
+	repo.MarkAsUpdating()
+	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	err = context.CloseDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	// Catch ^C
+	sigch := make(chan os.Signal)
+	signal.Notify(sigch, os.Interrupt)
+
+	count := len(queue)
+	context.Progress().Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
+
+	// Download from the queue
+	context.Progress().InitBar(downloadSize, true)
+
+	// Download all package files
+	ch := make(chan error, count)
+
+	// In separate goroutine (to avoid blocking main), push queue to downloader
+	go func() {
+		for _, task := range queue {
+			context.Downloader().DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
+		}
+
+		// We don't need queue after this point
+		queue = nil
+	}()
+
+	// Wait for all downloads to finish
+	errors := make([]string, 0)
+
+	for count > 0 {
+		select {
+		case <-sigch:
+			signal.Stop(sigch)
+			return fmt.Errorf("unable to update: interrupted")
+		case err = <-ch:
+			if err != nil {
+				errors = append(errors, err.Error())
+			}
+			count--
+		}
+	}
+
+	context.Progress().ShutdownBar()
+	signal.Stop(sigch)
+
+	if len(errors) > 0 {
+		return fmt.Errorf("unable to update: download errors:\n  %s\n", strings.Join(errors, "\n  "))
+	}
+
+	err = context.ReOpenDatabase()
+	if err != nil {
+		return fmt.Errorf("unable to update: %s", err)
+	}
+
+	repo.FinalizeDownload()
 	err = context.CollectionFactory().RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -80,6 +183,7 @@ Example:
 		Flag: *flag.NewFlagSet("aptly-mirror-update", flag.ExitOnError),
 	}
 
+	cmd.Flag.Bool("force", false, "force update mirror even if it is locked by another process")
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")

cmd/package.go

@@ -0,0 +1,16 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdPackage() *commander.Command {
+	return &commander.Command{
+		UsageLine: "package",
+		Short:     "operations on packages",
+		Subcommands: []*commander.Command{
+			makeCmdPackageSearch(),
+			makeCmdPackageShow(),
+		},
+	}
+}

cmd/package_search.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func aptlyPackageSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdPackageSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageSearch,
+		UsageLine: "search <package-query>",
+		Short:     "search for packages matching query",
+		Long: `
+Command search displays list of packages in whole DB that match package query
+
+Example:
+
+    $ aptly package search '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-search", flag.ExitOnError),
+	}
+
+	return cmd
+}

cmd/package_show.go

@@ -0,0 +1,137 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"os"
+)
+
+func printReferencesTo(p *deb.Package) (err error) {
+	err = context.CollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+		err := context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  mirror %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+		err := context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+		if repo.RefList() != nil {
+			if repo.RefList().Has(p) {
+				fmt.Printf("  local repo %s\n", repo)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
+		err := context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+		if snapshot.RefList().Has(p) {
+			fmt.Printf("  snapshot %s\n", snapshot)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func aptlyPackageShow(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 1 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	q, err := query.Parse(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	withFiles := context.flags.Lookup("with-files").Value.Get().(bool)
+	withReferences := context.flags.Lookup("with-references").Value.Get().(bool)
+
+	w := bufio.NewWriter(os.Stdout)
+
+	result := q.Query(context.CollectionFactory().PackageCollection())
+
+	err = result.ForEach(func(p *deb.Package) error {
+		p.Stanza().WriteTo(w)
+		w.Flush()
+		fmt.Printf("\n")
+
+		if withFiles {
+			fmt.Printf("Files in the pool:\n")
+			for _, f := range p.Files() {
+				path, err := context.PackagePool().Path(f.Filename, f.Checksums.MD5)
+				if err != nil {
+					return err
+				}
+				fmt.Printf("  %s\n", path)
+			}
+			fmt.Printf("\n")
+		}
+
+		if withReferences {
+			fmt.Printf("References to package:\n")
+			printReferencesTo(p)
+			fmt.Printf("\n")
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("unable to show: %s", err)
+	}
+
+	return err
+}
+
+func makeCmdPackageShow() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyPackageShow,
+		UsageLine: "show <package-query>",
+		Short:     "show details about packages matcing query",
+		Long: `
+Command shows displays detailed meta-information about packages
+matching query. Information from Debian control file is displayed.
+Optionally information about package files and
+inclusion into mirrors/snapshots/local repos is shown.
+
+Example:
+
+    $ aptly package show nginx-light_1.2.1-2.2+wheezy2_i386'
+`,
+		Flag: *flag.NewFlagSet("aptly-package-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-files", false, "display information about files from package pool")
+	cmd.Flag.Bool("with-references", false, "display information about mirrors, snapshots and local repos referencing this package")
+
+	return cmd
+}

cmd/publish.go

@@ -8,13 +8,14 @@ import (
 )
 
 func getSigner(flags *flag.FlagSet) (utils.Signer, error) {
-	if flags.Lookup("skip-signing").Value.Get().(bool) || context.Config().GpgDisableSign {
+	if LookupOption(context.Config().GpgDisableSign, flags, "skip-signing") {
 		return nil, nil
 	}
 
 	signer := &utils.GpgSigner{}
 	signer.SetKey(flags.Lookup("gpg-key").Value.String())
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
+	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 
 	err := signer.Init()
 	if err != nil {

cmd/publish_list.go

@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
 		}
 
 		if raw {
-			published = append(published, fmt.Sprintf("%s %s", repo.Prefix, repo.Distribution))
+			published = append(published, fmt.Sprintf("%s %s", repo.StoragePrefix(), repo.Distribution))
 		} else {
 			published = append(published, repo.String())
 		}

cmd/publish_repo.go

@@ -37,6 +37,8 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")

cmd/publish_snapshot.go

@@ -199,6 +199,8 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("origin", "", "origin name to publish")
 	cmd.Flag.String("label", "", "label to publish")

cmd/publish_switch.go

@@ -131,6 +131,8 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")

cmd/publish_update.go

@@ -98,6 +98,8 @@ Example:
 	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
+	cmd.Flag.String("passphrase", "", "GPG passhprase for the key (warning: could be insecure)")
+	cmd.Flag.String("passphrase-file", "", "GPG passhprase-file for the key (warning: could be insecure)")
 	cmd.Flag.Bool("skip-signing", false, "don't sign Release files with GPG")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 

cmd/repo.go

@@ -20,6 +20,7 @@ func makeCmdRepo() *commander.Command {
 			makeCmdRepoRemove(),
 			makeCmdRepoShow(),
 			makeCmdRepoRename(),
+			makeCmdRepoSearch(),
 		},
 	}
 }

cmd/repo_add.go

@@ -40,6 +40,8 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load packages: %s", err)
 	}
 
+	forceReplace := context.flags.Lookup("force-replace").Value.Get().(bool)
+
 	packageFiles := []string{}
 	failedFiles := []string{}
 
@@ -59,14 +61,16 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 					return nil
 				}
 
-				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+				if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+					strings.HasSuffix(info.Name(), ".dsc") {
 					packageFiles = append(packageFiles, path)
 				}
 
 				return nil
 			})
 		} else {
-			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".dsc") {
+			if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
+				strings.HasSuffix(info.Name(), ".dsc") {
 				packageFiles = append(packageFiles, location)
 			} else {
 				context.Progress().ColoredPrintf("@y[!]@| @!Unknwon file extenstion: %s@|", location)
@@ -79,6 +83,10 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 	processedFiles := []string{}
 	sort.Strings(packageFiles)
 
+	if forceReplace {
+		list.PrepareIndex()
+	}
+
 	for _, file := range packageFiles {
 		var (
 			stanza deb.Stanza
@@ -87,6 +95,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 		candidateProcessedFiles := []string{}
 		isSourcePackage := strings.HasSuffix(file, ".dsc")
+		isUdebPackage := strings.HasSuffix(file, ".udeb")
 
 		if isSourcePackage {
 			stanza, err = deb.GetControlFileFromDsc(file, verifier)
@@ -99,7 +108,11 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			}
 		} else {
 			stanza, err = deb.GetControlFileFromDeb(file)
-			p = deb.NewPackageFromControlFile(stanza)
+			if isUdebPackage {
+				p = deb.NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = deb.NewPackageFromControlFile(stanza)
+			}
 		}
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to read file %s: %s@|", file, err)
@@ -155,6 +168,14 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 			continue
 		}
 
+		if forceReplace {
+			conflictingPackages := list.Search(deb.Dependency{Pkg: p.Name, Version: p.Version, Architecture: p.Architecture}, true)
+			for _, cp := range conflictingPackages {
+				context.Progress().ColoredPrintf("@r[-]@| %s removed due to conflict with package being added", cp)
+				list.Remove(cp)
+			}
+		}
+
 		err = list.Add(p)
 		if err != nil {
 			context.Progress().ColoredPrintf("@y[!]@| @!Unable to add package to repo %s: %s@|", p, err)
@@ -202,8 +223,8 @@ func makeCmdRepoAdd() *commander.Command {
 		UsageLine: "add <name> <package file.deb>|<directory> ...",
 		Short:     "add packages to local repository",
 		Long: `
-Command adds packages to local repository from .deb (binary packages) and .dsc (source packages) files.
-When importing from directory aptly would do recursive scan looking for all files matching *.deb or *.dsc
+Command adds packages to local repository from .deb, .udeb (binary packages) and .dsc (source packages) files.
+When importing from directory aptly would do recursive scan looking for all files matching *.[u]deb or *.dsc
 patterns. Every file discovered would be analyzed to extract metadata, package would then be created and added
 to the database. Files would be imported to internal package pool. For source packages, all required files are
 added automatically as well. Extra files for source package should be in the same directory as *.dsc file.
@@ -216,6 +237,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("remove-files", false, "remove files that have been imported successfully into repository")
+	cmd.Flag.Bool("force-replace", false, "when adding package that conflicts with existing package, remove existing package")
 
 	return cmd
 }

cmd/repo_edit.go

@@ -50,7 +50,7 @@ func makeCmdRepoEdit() *commander.Command {
 		UsageLine: "edit <name>",
 		Short:     "edit properties of local repository",
 		Long: `
-Command edit allows to change metadata of local repository:
+Command edit allows one to change metadata of local repository:
 comment, default distribution and component.
 
 Example:

cmd/repo_search.go

@@ -0,0 +1,26 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+)
+
+func makeCmdRepoSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search repo for packages matching query",
+		Long: `
+Command search displays list of packages in local repository that match package query
+
+Example:
+
+    $ aptly repo search my-software '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-repo-show", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/run.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/commander"
+)
+
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+	defer func() {
+		if r := recover(); r != nil {
+			fatal, ok := r.(*FatalError)
+			if !ok {
+				panic(r)
+			}
+			fmt.Println("ERROR:", fatal.Message)
+			returnCode = fatal.ReturnCode
+		}
+	}()
+
+	returnCode = 0
+
+	flags, args, err := cmd.ParseFlags(cmdArgs)
+	if err != nil {
+		Fatal(err)
+	}
+
+	if initContext {
+		err = InitContext(flags)
+		if err != nil {
+			Fatal(err)
+		}
+		defer ShutdownContext()
+	}
+
+	context.UpdateFlags(flags)
+
+	err = cmd.Dispatch(args)
+	if err != nil {
+		Fatal(err)
+	}
+
+	return
+}

cmd/snapshot.go

@@ -18,6 +18,8 @@ func makeCmdSnapshot() *commander.Command {
 			makeCmdSnapshotMerge(),
 			makeCmdSnapshotDrop(),
 			makeCmdSnapshotRename(),
+			makeCmdSnapshotSearch(),
+			makeCmdSnapshotFilter(),
 		},
 	}
 }

cmd/snapshot_create.go

@@ -23,6 +23,11 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
+		err = repo.CheckLock()
+		if err != nil {
+			return fmt.Errorf("unable to create snapshot: %s", err)
+		}
+
 		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)

cmd/snapshot_filter.go

@@ -0,0 +1,107 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+	"strings"
+)
+
+func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) < 3 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+
+	// Load <source> snapshot
+	source, err := context.CollectionFactory().SnapshotCollection().ByName(args[0])
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	err = context.CollectionFactory().SnapshotCollection().LoadComplete(source)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Convert snapshot to package list
+	context.Progress().Printf("Loading packages (%d)...\n", source.RefList().Len())
+	packageList, err := deb.NewPackageListFromRefList(source.RefList(), context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to load packages: %s", err)
+	}
+
+	context.Progress().Printf("Building indexes...\n")
+	packageList.PrepareIndex()
+
+	// Calculate architectures
+	var architecturesList []string
+
+	if len(context.ArchitecturesList()) > 0 {
+		architecturesList = context.ArchitecturesList()
+	} else {
+		architecturesList = packageList.Architectures(false)
+	}
+
+	sort.Strings(architecturesList)
+
+	if len(architecturesList) == 0 && withDeps {
+		return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+	}
+
+	// Initial queries out of arguments
+	queries := make([]deb.PackageQuery, len(args)-2)
+	for i, arg := range args[2:] {
+		queries[i], err = query.Parse(arg)
+		if err != nil {
+			return fmt.Errorf("unable to parse query: %s", err)
+		}
+	}
+
+	// Filter with dependencies as requested
+	result, err := packageList.Filter(queries, withDeps, nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to filter: %s", err)
+	}
+
+	// Create <destination> snapshot
+	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
+		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
+
+	err = context.CollectionFactory().SnapshotCollection().Add(destination)
+	if err != nil {
+		return fmt.Errorf("unable to create snapshot: %s", err)
+	}
+
+	context.Progress().Printf("\nSnapshot %s successfully filtered.\nYou can run 'aptly publish snapshot %s' to publish snapshot as Debian repository.\n", destination.Name, destination.Name)
+
+	return err
+}
+
+func makeCmdSnapshotFilter() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotFilter,
+		UsageLine: "filter <source> <destination> <package-query> ...",
+		Short:     "filter packages in snapshot producing another snapshot",
+		Long: `
+Command filter does filtering in snapshot <source>, producing another
+snapshot <destination>. Packages could be specified simply
+as 'package-name' or as package queries.
+
+Example:
+
+    $ aptly snapshot filter wheezy-main wheezy-required 'Priorioty (required)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-filter", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependent packages as well")
+
+	return cmd
+}

cmd/snapshot_search.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"fmt"
+	"github.com/smira/aptly/deb"
+	"github.com/smira/aptly/query"
+	"github.com/smira/commander"
+	"github.com/smira/flag"
+	"sort"
+)
+
+func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error {
+	var err error
+	if len(args) != 2 {
+		cmd.Usage()
+		return commander.ErrCommandError
+	}
+
+	name := args[0]
+	command := cmd.Parent.Name()
+
+	var reflist *deb.PackageRefList
+
+	if command == "snapshot" {
+		snapshot, err := context.CollectionFactory().SnapshotCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().SnapshotCollection().LoadComplete(snapshot)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = snapshot.RefList()
+	} else if command == "mirror" {
+		repo, err := context.CollectionFactory().RemoteRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().RemoteRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else if command == "repo" {
+		repo, err := context.CollectionFactory().LocalRepoCollection().ByName(name)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		err = context.CollectionFactory().LocalRepoCollection().LoadComplete(repo)
+		if err != nil {
+			return fmt.Errorf("unable to search: %s", err)
+		}
+
+		reflist = repo.RefList()
+	} else {
+		panic("unknown command")
+	}
+
+	list, err := deb.NewPackageListFromRefList(reflist, context.CollectionFactory().PackageCollection(), context.Progress())
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	list.PrepareIndex()
+
+	q, err := query.Parse(args[1])
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	withDeps := context.flags.Lookup("with-deps").Value.Get().(bool)
+	architecturesList := []string{}
+
+	if withDeps {
+		if len(context.ArchitecturesList()) > 0 {
+			architecturesList = context.ArchitecturesList()
+		} else {
+			architecturesList = list.Architectures(false)
+		}
+
+		sort.Strings(architecturesList)
+
+		if len(architecturesList) == 0 {
+			return fmt.Errorf("unable to determine list of architectures, please specify explicitly")
+		}
+	}
+
+	result, err := list.Filter([]deb.PackageQuery{q}, withDeps,
+		nil, context.DependencyOptions(), architecturesList)
+	if err != nil {
+		return fmt.Errorf("unable to search: %s", err)
+	}
+
+	result.ForEach(func(p *deb.Package) error {
+		context.Progress().Printf("%s\n", p)
+		return nil
+	})
+
+	return err
+}
+
+func makeCmdSnapshotSearch() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlySnapshotMirrorRepoSearch,
+		UsageLine: "search <name> <package-query>",
+		Short:     "search snapshot for packages matching query",
+		Long: `
+Command search displays list of packages in snapshot that match package query
+
+Example:
+
+    $ aptly snapshot search wheezy-main '$Architecture (i386), Name (% *-dev)'
+`,
+		Flag: *flag.NewFlagSet("aptly-snapshot-search", flag.ExitOnError),
+	}
+
+	cmd.Flag.Bool("with-deps", false, "include dependencies into search results")
+
+	return cmd
+}

cmd/task.go

@@ -0,0 +1,15 @@
+package cmd
+
+import (
+	"github.com/smira/commander"
+)
+
+func makeCmdTask() *commander.Command {
+	return &commander.Command{
+		UsageLine: "task",
+		Short:     "manage aptly tasks",
+		Subcommands: []*commander.Command{
+			makeCmdTaskRun(),
+		},
+	}
+}

cmd/task_run.go

@@ -0,0 +1,148 @@
+package cmd
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/mattn/go-shellwords"
+	"github.com/smira/commander"
+)
+
+func aptlyTaskRun(cmd *commander.Command, args []string) error {
+	var err error
+	var cmdList [][]string
+
+	if filename := cmd.Flag.Lookup("filename").Value.Get().(string); filename != "" {
+		var text string
+		cmdArgs := []string{}
+
+		if finfo, err := os.Stat(filename); os.IsNotExist(err) || finfo.IsDir() {
+			return fmt.Errorf("no such file, %s\n", filename)
+		}
+
+		fmt.Println("Reading file...\n")
+
+		file, err := os.Open(filename)
+
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+
+		for scanner.Scan() {
+			text = strings.TrimSpace(scanner.Text()) + ","
+			parsedArgs, _ := shellwords.Parse(text)
+			cmdArgs = append(cmdArgs, parsedArgs...)
+		}
+
+		if err = scanner.Err(); err != nil {
+			return err
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("the file is empty")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else if len(args) == 0 {
+		var text string
+		cmdArgs := []string{}
+
+		fmt.Println("Please enter one command per line and leave one blank when finished.")
+
+		reader := bufio.NewReader(os.Stdin)
+		for {
+			fmt.Printf("> ")
+			text, _ = reader.ReadString('\n')
+			if text == "\n" {
+				break
+			} else {
+				text = strings.TrimSpace(text) + ","
+				parsedArgs, _ := shellwords.Parse(text)
+				cmdArgs = append(cmdArgs, parsedArgs...)
+			}
+		}
+
+		if len(cmdArgs) == 0 {
+			return fmt.Errorf("nothing entered")
+		}
+
+		cmdList = formatCommands(cmdArgs)
+	} else {
+		cmdList = formatCommands(args)
+	}
+
+	commandErrored := false
+
+	for i, command := range cmdList {
+		if !commandErrored {
+			context.Progress().ColoredPrintf("@g%d) [Running]: %s@!", (i + 1), strings.Join(command, " "))
+			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
+			context.Progress().Flush()
+
+			returnCode := Run(RootCommand(), command, false)
+			if returnCode != 0 {
+				commandErrored = true
+			}
+			context.Progress().ColoredPrintf("\n@yEnd command output: ------------------------------@!")
+			CleanupContext()
+		} else {
+			context.Progress().ColoredPrintf("@r%d) [Skipping]: %s@!", (i + 1), strings.Join(command, " "))
+		}
+	}
+
+	if commandErrored {
+		err = fmt.Errorf("at least one command has reported an error")
+	}
+
+	return err
+}
+
+func formatCommands(args []string) [][]string {
+	var cmd []string
+	var cmdArray [][]string
+
+	for _, s := range args {
+		if sTrimmed := strings.TrimRight(s, ","); sTrimmed != s {
+			cmd = append(cmd, sTrimmed)
+			cmdArray = append(cmdArray, cmd)
+			cmd = []string{}
+		} else {
+			cmd = append(cmd, s)
+		}
+	}
+
+	if len(cmd) > 0 {
+		cmdArray = append(cmdArray, cmd)
+	}
+
+	return cmdArray
+}
+
+func makeCmdTaskRun() *commander.Command {
+	cmd := &commander.Command{
+		Run:       aptlyTaskRun,
+		UsageLine: "run -filename=<filename> | <command1>, <command2>, ...",
+		Short:     "run aptly tasks",
+		Long: `
+Command helps origanise multiple aptly commands in one single aptly task, running as single thread.
+
+Example:
+
+  $ aptly task run
+  > repo create local
+  > repo add local pkg1
+  > publish repo local
+  > serve
+  >
+
+`,
+	}
+
+	cmd.Flag.String("filename", "", "specifies the filename that contains the commands to run")
+	return cmd
+}

database/leveldb.go

@@ -24,12 +24,14 @@ type Storage interface {
 	KeysByPrefix(prefix []byte) [][]byte
 	FetchByPrefix(prefix []byte) [][]byte
 	Close() error
+	ReOpen() error
 	StartBatch()
 	FinishBatch() error
 	CompactDB() error
 }
 
 type levelDB struct {
+	path  string
 	db    *leveldb.DB
 	batch *leveldb.Batch
 }
@@ -39,17 +41,21 @@ var (
 	_ Storage = &levelDB{}
 )
 
-// OpenDB opens (creates) LevelDB database
-func OpenDB(path string) (Storage, error) {
+func internalOpen(path string) (*leveldb.DB, error) {
 	o := &opt.Options{
 		Filter: filter.NewBloomFilter(10),
 	}
 
-	db, err := leveldb.OpenFile(path, o)
+	return leveldb.OpenFile(path, o)
+}
+
+// OpenDB opens (creates) LevelDB database
+func OpenDB(path string) (Storage, error) {
+	db, err := internalOpen(path)
 	if err != nil {
 		return nil, err
 	}
-	return &levelDB{db: db}, nil
+	return &levelDB{db: db, path: path}, nil
 }
 
 // RecoverDB recovers LevelDB database from corruption
@@ -147,7 +153,23 @@ func (l *levelDB) FetchByPrefix(prefix []byte) [][]byte {
 
 // Close finishes DB work
 func (l *levelDB) Close() error {
-	return l.db.Close()
+	if l.db == nil {
+		return nil
+	}
+	err := l.db.Close()
+	l.db = nil
+	return err
+}
+
+// Reopen tries to re-open the database
+func (l *levelDB) ReOpen() error {
+	if l.db != nil {
+		return nil
+	}
+
+	var err error
+	l.db, err = internalOpen(l.path)
+	return err
 }
 
 // StartBatch starts batch processing of keys

database/leveldb_test.go

@@ -155,3 +155,23 @@ func (s *LevelDBSuite) TestCompactDB(c *C) {
 
 	c.Check(s.db.CompactDB(), IsNil)
 }
+
+func (s *LevelDBSuite) TestReOpen(c *C) {
+	var (
+		key   = []byte("key")
+		value = []byte("value")
+	)
+
+	err := s.db.Put(key, value)
+	c.Assert(err, IsNil)
+
+	err = s.db.Close()
+	c.Assert(err, IsNil)
+
+	err = s.db.ReOpen()
+	c.Assert(err, IsNil)
+
+	result, err := s.db.Get(key)
+	c.Assert(err, IsNil)
+	c.Assert(result, DeepEquals, value)
+}

deb/index_files.go

@@ -0,0 +1,254 @@
+package deb
+
+import (
+	"bufio"
+	"fmt"
+	"github.com/smira/aptly/aptly"
+	"github.com/smira/aptly/utils"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type indexFiles struct {
+	publishedStorage aptly.PublishedStorage
+	basePath         string
+	renameMap        map[string]string
+	generatedFiles   map[string]utils.ChecksumInfo
+	tempDir          string
+	suffix           string
+	indexes          map[string]*indexFile
+}
+
+type indexFile struct {
+	parent       *indexFiles
+	discardable  bool
+	compressable bool
+	signable     bool
+	relativePath string
+	tempFilename string
+	tempFile     *os.File
+	w            *bufio.Writer
+}
+
+func (file *indexFile) BufWriter() (*bufio.Writer, error) {
+	if file.w == nil {
+		var err error
+		file.tempFilename = filepath.Join(file.parent.tempDir, strings.Replace(file.relativePath, "/", "_", -1))
+		file.tempFile, err = os.Create(file.tempFilename)
+		if err != nil {
+			return nil, fmt.Errorf("unable to create temporary index file: %s", err)
+		}
+
+		file.w = bufio.NewWriter(file.tempFile)
+	}
+
+	return file.w, nil
+}
+
+func (file *indexFile) Finalize(signer utils.Signer) error {
+	if file.w == nil {
+		if file.discardable {
+			return nil
+		}
+		file.BufWriter()
+	}
+
+	err := file.w.Flush()
+	if err != nil {
+		file.tempFile.Close()
+		return fmt.Errorf("unable to write to index file: %s", err)
+	}
+
+	if file.compressable {
+		err = utils.CompressFile(file.tempFile)
+		if err != nil {
+			file.tempFile.Close()
+			return fmt.Errorf("unable to compress index file: %s", err)
+		}
+	}
+
+	file.tempFile.Close()
+
+	exts := []string{""}
+	if file.compressable {
+		exts = append(exts, ".gz", ".bz2")
+	}
+
+	for _, ext := range exts {
+		var checksumInfo utils.ChecksumInfo
+
+		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
+		if err != nil {
+			return fmt.Errorf("unable to collect checksums: %s", err)
+		}
+		file.parent.generatedFiles[file.relativePath+ext] = checksumInfo
+	}
+
+	err = file.parent.publishedStorage.MkDir(filepath.Dir(filepath.Join(file.parent.basePath, file.relativePath)))
+	if err != nil {
+		return fmt.Errorf("unable to create dir: %s", err)
+	}
+
+	for _, ext := range exts {
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext),
+			file.tempFilename+ext)
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+ext)] =
+				filepath.Join(file.parent.basePath, file.relativePath+ext)
+		}
+	}
+
+	if file.signable && signer != nil {
+		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to detached sign file: %s", err)
+		}
+
+		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to clearsign file: %s", err)
+		}
+
+		if file.parent.suffix != "" {
+			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+				filepath.Join(file.parent.basePath, "In"+file.relativePath)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+			file.tempFilename+".gpg")
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+
+		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
+			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+		if err != nil {
+			return fmt.Errorf("unable to publish file: %s", err)
+		}
+	}
+
+	return nil
+}
+
+func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string) *indexFiles {
+	return &indexFiles{
+		publishedStorage: publishedStorage,
+		basePath:         basePath,
+		renameMap:        make(map[string]string),
+		generatedFiles:   make(map[string]utils.ChecksumInfo),
+		tempDir:          tempDir,
+		suffix:           suffix,
+		indexes:          make(map[string]*indexFile),
+	}
+}
+
+func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("pi-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Sources")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  false,
+			compressable: true,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexFile {
+	key := fmt.Sprintf("ri-%s-%s-%s", component, arch, udeb)
+	file, ok := files.indexes[key]
+	if !ok {
+		var relativePath string
+
+		if arch == "source" {
+			relativePath = filepath.Join(component, "source", "Release")
+		} else {
+			if udeb {
+				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Release")
+			} else {
+				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+			}
+		}
+
+		file = &indexFile{
+			parent:       files,
+			discardable:  udeb,
+			compressable: false,
+			signable:     false,
+			relativePath: relativePath,
+		}
+
+		files.indexes[key] = file
+	}
+
+	return file
+}
+
+func (files *indexFiles) ReleaseFile() *indexFile {
+	return &indexFile{
+		parent:       files,
+		discardable:  false,
+		compressable: false,
+		signable:     true,
+		relativePath: "Release",
+	}
+}
+
+func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+	if progress != nil {
+		progress.InitBar(int64(len(files.indexes)), false)
+		defer progress.ShutdownBar()
+	}
+
+	for _, file := range files.indexes {
+		err = file.Finalize(nil)
+		if err != nil {
+			return
+		}
+		if progress != nil {
+			progress.AddBar(1)
+		}
+	}
+
+	files.indexes = make(map[string]*indexFile)
+
+	return
+}
+
+func (files *indexFiles) RenameFiles() error {
+	var err error
+
+	for oldName, newName := range files.renameMap {
+		err = files.publishedStorage.RenameFile(oldName, newName)
+		if err != nil {
+			return fmt.Errorf("unable to rename: %s", err)
+		}
+	}
+
+	return nil
+}

deb/list.go

@@ -41,6 +41,7 @@ type PackageList struct {
 // Verify interface
 var (
 	_ sort.Interface = &PackageList{}
+	_ PackageCatalog = &PackageList{}
 )
 
 // NewPackageList creates empty package list
@@ -235,6 +236,7 @@ func depSliceDeduplicate(s []Dependency) []Dependency {
 //
 // Analysis would be peformed for each architecture, in specified sources
 func (l *PackageList) VerifyDependencies(options int, architectures []string, sources *PackageList, progress aptly.Progress) ([]Dependency, error) {
+	l.PrepareIndex()
 	missing := make([]Dependency, 0, 128)
 
 	if progress != nil {
@@ -244,7 +246,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 	for _, arch := range architectures {
 		cache := make(map[string]bool, 2048)
 
-		for _, p := range l.packages {
+		for _, p := range l.packagesIndex {
 			if progress != nil {
 				progress.AddBar(1)
 			}
@@ -262,7 +264,6 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 				variants = depSliceDeduplicate(variants)
 
 				variantsMissing := make([]Dependency, 0, len(variants))
-				missingCount := 0
 
 				for _, dep := range variants {
 					if dep.Architecture == "" {
@@ -270,35 +271,23 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
 					}
 
 					hash := dep.Hash()
-					r, ok := cache[hash]
-					if ok {
-						if !r {
-							missingCount++
-						}
-						continue
+					satisfied, ok := cache[hash]
+					if !ok {
+						satisfied = sources.Search(dep, false) != nil
+						cache[hash] = satisfied
 					}
 
-					if sources.Search(dep, false) == nil {
+					if !satisfied && !ok {
 						variantsMissing = append(variantsMissing, dep)
-						missingCount++
-					} else {
-						cache[hash] = true
+					}
+
+					if satisfied && options&DepFollowAllVariants == 0 {
+						variantsMissing = nil
+						break
 					}
 				}
 
-				if options&DepFollowAllVariants == DepFollowAllVariants {
-					missing = append(missing, variantsMissing...)
-					for _, dep := range variantsMissing {
-						cache[dep.Hash()] = false
-					}
-				} else {
-					if missingCount == len(variants) {
-						missing = append(missing, variantsMissing...)
-						for _, dep := range variantsMissing {
-							cache[dep.Hash()] = false
-						}
-					}
-				}
+				missing = append(missing, variantsMissing...)
 			}
 		}
 	}
@@ -334,6 +323,10 @@ func (l *PackageList) Less(i, j int) bool {
 
 // PrepareIndex prepares list for indexing
 func (l *PackageList) PrepareIndex() {
+	if l.indexed {
+		return
+	}
+
 	l.packagesIndex = make([]*Package, l.Len())
 	l.providesIndex = make(map[string][]*Package, 128)
 
@@ -364,6 +357,23 @@ func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
 	return
 }
 
+// SearchSupported returns true for PackageList
+func (l *PackageList) SearchSupported() bool {
+	return true
+}
+
+// SearchByKey looks up package by exact key reference
+func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	pkg := l.packages["P"+arch+" "+name+" "+version]
+	if pkg != nil {
+		result.Add(pkg)
+	}
+
+	return
+}
+
 // Search searches package index for specified package(s) using optimized queries
 func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
 	if !l.indexed {
@@ -414,6 +424,7 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 
 	if withDependencies {
 		added := result.Len()
+		result.PrepareIndex()
 
 		dependencySource := NewPackageList()
 		if source != nil {
@@ -434,12 +445,21 @@ func (l *PackageList) Filter(queries []PackageQuery, withDependencies bool, sour
 
 			// try to satisfy dependencies
 			for _, dep := range missing {
+				// dependency might have already been satisfied
+				// with packages already been added
+				if result.Search(dep, false) != nil {
+					continue
+				}
+
 				searchResults := l.Search(dep, false)
 				if searchResults != nil {
 					for _, p := range searchResults {
 						result.Add(p)
 						dependencySource.Add(p)
 						added++
+						if dependencyOptions&DepFollowAllVariants == 0 {
+							break
+						}
 					}
 				}
 			}

deb/package.go

@@ -24,6 +24,8 @@ type Package struct {
 	Provides []string
 	// Is this source package
 	IsSource bool
+	// Is this udeb package
+	IsUdeb bool
 	// Hash of files section
 	FilesHash uint64
 	// Is this >= 0.6 package?
@@ -169,6 +171,14 @@ func NewSourcePackageFromControlFile(input Stanza) (*Package, error) {
 	return result, nil
 }
 
+// NewUdebPackageFromControlFile creates .udeb Package from parsed Debian control file
+func NewUdebPackageFromControlFile(input Stanza) *Package {
+	p := NewPackageFromControlFile(input)
+	p.IsUdeb = true
+
+	return p
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -220,6 +230,9 @@ func (p *Package) GetField(name string) string {
 		if p.IsSource {
 			return "source"
 		}
+		if p.IsUdeb {
+			return "udeb"
+		}
 		return "deb"
 	case "Name":
 		return p.Name

deb/package_collection.go

@@ -15,6 +15,11 @@ type PackageCollection struct {
 	codecHandle  *codec.MsgpackHandle
 }
 
+// Verify interface
+var (
+	_ PackageCatalog = &PackageCollection{}
+)
+
 // NewPackageCollection creates new PackageCollection and binds it to database
 func NewPackageCollection(db database.Storage) *PackageCollection {
 	return &PackageCollection{
@@ -237,3 +242,49 @@ func (collection *PackageCollection) DeleteByKey(key []byte) error {
 	}
 	return nil
 }
+
+// Scan does full scan on all the packages
+func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte("P")) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if q.Matches(pkg) {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}
+
+// Search is not implemented
+func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
+	panic("Not implemented")
+}
+
+// SearchSupported returns false
+func (collection *PackageCollection) SearchSupported() bool {
+	return false
+}
+
+// SearchByKey finds package by exact key
+func (collection *PackageCollection) SearchByKey(arch, name, version string) (result *PackageList) {
+	result = NewPackageList()
+
+	for _, key := range collection.db.KeysByPrefix([]byte(fmt.Sprintf("P%s %s %s", arch, name, version))) {
+		pkg, err := collection.ByKey(key)
+		if err != nil {
+			panic(fmt.Sprintf("unable to load package: %s", err))
+		}
+
+		if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
+			result.Add(pkg)
+		}
+	}
+
+	return
+}

deb/package_test.go

@@ -28,6 +28,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
 	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "alien-arena-common")
 	c.Check(p.Version, Equals, "7.40-2")
 	c.Check(p.Architecture, Equals, "i386")
@@ -40,11 +41,27 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6 (>= 2.7)", "alien-arena-data (>= 7.40)"})
 }
 
+func (s *PackageSuite) TestNewUdebFromPara(c *C) {
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p := NewUdebPackageFromControlFile(stanza)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, true)
+	c.Check(p.Name, Equals, "dmidecode-udeb")
+	c.Check(p.Version, Equals, "2.11-9")
+	c.Check(p.Architecture, Equals, "amd64")
+	c.Check(p.Provides, DeepEquals, []string(nil))
+	c.Check(p.Files(), HasLen, 1)
+	c.Check(p.Files()[0].Filename, Equals, "dmidecode-udeb_2.11-9_amd64.udeb")
+	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
 	c.Check(err, IsNil)
 	c.Check(p.IsSource, Equals, true)
+	c.Check(p.IsUdeb, Equals, false)
 	c.Check(p.Name, Equals, "access-modifier-checker")
 	c.Check(p.Version, Equals, "1.0-4")
 	c.Check(p.Architecture, Equals, "source")
@@ -134,21 +151,28 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza()
+	p5 := NewUdebPackageFromControlFile(stanza5)
+
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
 	c.Check(p2.GetField("$Source"), Equals, "alien-arena-common")
 	c.Check(p3.GetField("$Source"), Equals, "alien-arena")
 	c.Check(p4.GetField("$Source"), Equals, "")
+	c.Check(p5.GetField("$Source"), Equals, "dmidecode")
 
 	c.Check(p.GetField("$SourceVersion"), Equals, "7.40-2")
 	c.Check(p2.GetField("$SourceVersion"), Equals, "7.40-2")
 	c.Check(p3.GetField("$SourceVersion"), Equals, "3.5")
 	c.Check(p4.GetField("$SourceVersion"), Equals, "")
+	c.Check(p5.GetField("$SourceVersion"), Equals, "2.11-9")
 
 	c.Check(p.GetField("$Architecture"), Equals, "i386")
 	c.Check(p4.GetField("$Architecture"), Equals, "source")
+	c.Check(p5.GetField("$Architecture"), Equals, "amd64")
 
 	c.Check(p.GetField("$PackageType"), Equals, "deb")
 	c.Check(p4.GetField("$PackageType"), Equals, "source")
+	c.Check(p5.GetField("$PackageType"), Equals, "udeb")
 
 	c.Check(p.GetField("Name"), Equals, "alien-arena-common")
 	c.Check(p4.GetField("Name"), Equals, "access-modifier-checker")
@@ -455,3 +479,20 @@ Directory: pool/main/a/access-modifier-checker
 Priority: source
 Section: java
 `
+
+const udebPackageMeta = `Package: dmidecode-udeb
+Source: dmidecode
+Version: 2.11-9
+Installed-Size: 115
+Maintainer: Daniel Baumann <daniel.baumann@progress-technologies.net>
+Architecture: amd64
+Depends: libc6-udeb (>= 2.13)
+Description: SMBIOS/DMI table decoder (udeb)
+Description-md5: bdfb786c6a57097be8c8600b800e749f
+Section: debian-installer
+Priority: optional
+Filename: pool/main/d/dmidecode/dmidecode-udeb_2.11-9_amd64.udeb
+Size: 29188
+MD5sum: ae70341c4d96dcded89fa670bcfea31e
+SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
+SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`

deb/publish.go

@@ -1,7 +1,6 @@
 package deb
 
 import (
-	"bufio"
 	"bytes"
 	"code.google.com/p/go-uuid/uuid"
 	"fmt"
@@ -169,6 +168,9 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		if distribution == "" || component == "" {
 			rootDistributions, rootComponents := walkUpTree(source, collectionFactory)
 			if distribution == "" {
+				for i := range rootDistributions {
+					rootDistributions[i] = strings.Replace(rootDistributions[i], "/", "-", -1)
+				}
 				discoveredDistributions = append(discoveredDistributions, rootDistributions...)
 			}
 			if component == "" {
@@ -228,6 +230,10 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
 		}
 	}
 
+	if strings.Index(distribution, "/") != -1 {
+		return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
+	}
+
 	result.Distribution = distribution
 
 	return result, nil
@@ -440,9 +446,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		suffix = ".tmp"
 	}
 
-	generatedFiles := map[string]utils.ChecksumInfo{}
-	renameMap := map[string]string{}
-
 	if progress != nil {
 		progress.Printf("Generating metadata files and linking package files...\n")
 	}
@@ -454,41 +457,44 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	}
 	defer os.RemoveAll(tempDir)
 
+	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix)
+
 	for component, list := range lists {
-		var relativePath string
+		hadUdebs := false
 
-		// For all architectures, generate packages/sources files
+		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
+			indexes.PackageIndex(component, arch, false)
+		}
+
+		if progress != nil {
+			progress.InitBar(int64(list.Len()), false)
+		}
+
+		err = list.ForEach(func(pkg *Package) error {
 			if progress != nil {
-				progress.InitBar(int64(list.Len()), false)
+				progress.AddBar(1)
 			}
 
-			if arch == "source" {
-				relativePath = filepath.Join(component, "source", "Sources")
-			} else {
-				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
-			}
-			err = publishedStorage.MkDir(filepath.Dir(filepath.Join(basePath, relativePath)))
-			if err != nil {
-				return err
-			}
-
-			var packagesFile *os.File
-
-			packagesFileName := filepath.Join(tempDir, fmt.Sprintf("pkgs_%s_%s", component, arch))
-			packagesFile, err = os.Create(packagesFileName)
-			if err != nil {
-				return fmt.Errorf("unable to create temporary Packages file: %s", err)
-			}
-
-			bufWriter := bufio.NewWriter(packagesFile)
-
-			err = list.ForEach(func(pkg *Package) error {
-				if progress != nil {
-					progress.AddBar(1)
-				}
+			matches := false
+			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
+					matches = true
+					break
+				}
+			}
+
+			if matches {
+				hadUdebs = hadUdebs || pkg.IsUdeb
+				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
+				if err != nil {
+					return err
+				}
+			}
+
+			for _, arch := range p.Architectures {
+				if pkg.MatchesArchitecture(arch) {
+					bufWriter, err := indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
 					if err != nil {
 						return err
 					}
@@ -501,113 +507,63 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					if err != nil {
 						return err
 					}
-
-					pkg.files = nil
-					pkg.deps = nil
-					pkg.extra = nil
-
-				}
-
-				return nil
-			})
-
-			if err != nil {
-				return fmt.Errorf("unable to process packages: %s", err)
-			}
-
-			err = bufWriter.Flush()
-			if err != nil {
-				return fmt.Errorf("unable to write Packages file: %s", err)
-			}
-
-			err = utils.CompressFile(packagesFile)
-			if err != nil {
-				return fmt.Errorf("unable to compress Packages files: %s", err)
-			}
-
-			packagesFile.Close()
-
-			for _, ext := range []string{"", ".gz", ".bz2"} {
-				var checksumInfo utils.ChecksumInfo
-
-				checksumInfo, err = utils.ChecksumsForFile(packagesFileName + ext)
-				if err != nil {
-					return fmt.Errorf("unable to collect checksums: %s", err)
-				}
-				generatedFiles[relativePath+ext] = checksumInfo
-
-				err = publishedStorage.PutFile(filepath.Join(basePath, relativePath+suffix+ext), packagesFileName+ext)
-				if err != nil {
-					return fmt.Errorf("unable to publish file: %s", err)
-				}
-
-				if suffix != "" {
-					renameMap[filepath.Join(basePath, relativePath+suffix+ext)] = filepath.Join(basePath, relativePath+ext)
 				}
 			}
 
-			if progress != nil {
-				progress.ShutdownBar()
+			pkg.files = nil
+			pkg.deps = nil
+			pkg.extra = nil
+
+			return nil
+		})
+
+		if err != nil {
+			return fmt.Errorf("unable to process packages: %s", err)
+		}
+
+		if progress != nil {
+			progress.ShutdownBar()
+		}
+
+		udebs := []bool{false}
+		if hadUdebs {
+			udebs = append(udebs, true)
+
+			// For all architectures, pregenerate .udeb indexes
+			for _, arch := range p.Architectures {
+				indexes.PackageIndex(component, arch, true)
 			}
 		}
 
 		// For all architectures, generate Release files
 		for _, arch := range p.Architectures {
-			release := make(Stanza)
-			release["Archive"] = p.Distribution
-			release["Architecture"] = arch
-			release["Component"] = component
-			release["Origin"] = p.GetOrigin()
-			release["Label"] = p.GetLabel()
+			for _, udeb := range udebs {
+				release := make(Stanza)
+				release["Archive"] = p.Distribution
+				release["Architecture"] = arch
+				release["Component"] = component
+				release["Origin"] = p.GetOrigin()
+				release["Label"] = p.GetLabel()
 
-			if arch == "source" {
-				relativePath = filepath.Join(component, "source", "Release")
-			} else {
-				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Release")
+				bufWriter, err := indexes.ReleaseIndex(component, arch, udeb).BufWriter()
+
+				err = release.WriteTo(bufWriter)
+				if err != nil {
+					return fmt.Errorf("unable to create Release file: %s", err)
+				}
 			}
-
-			var file *os.File
-
-			fileName := filepath.Join(tempDir, fmt.Sprintf("release_%s_%s", component, arch))
-			file, err = os.Create(fileName)
-			if err != nil {
-				return fmt.Errorf("unable to create temporary Release file: %s", err)
-			}
-
-			bufWriter := bufio.NewWriter(file)
-
-			err = release.WriteTo(bufWriter)
-			if err != nil {
-				return fmt.Errorf("unable to create Release file: %s", err)
-			}
-
-			err = bufWriter.Flush()
-			if err != nil {
-				return fmt.Errorf("unable to create Release file: %s", err)
-			}
-
-			file.Close()
-
-			var checksumInfo utils.ChecksumInfo
-			checksumInfo, err = utils.ChecksumsForFile(fileName)
-			if err != nil {
-				return fmt.Errorf("unable to collect checksums: %s", err)
-			}
-			generatedFiles[relativePath] = checksumInfo
-
-			err = publishedStorage.PutFile(filepath.Join(basePath, relativePath+suffix), fileName)
-			if err != nil {
-				file.Close()
-				return fmt.Errorf("unable to publish file: %s", err)
-			}
-
-			if suffix != "" {
-				renameMap[filepath.Join(basePath, relativePath+suffix)] = filepath.Join(basePath, relativePath)
-			}
-
 		}
 	}
 
+	if progress != nil {
+		progress.Printf("Finalizing metadata files...\n")
+	}
+
+	err = indexes.FinalizeAll(progress)
+	if err != nil {
+		return err
+	}
+
 	release := make(Stanza)
 	release["Origin"] = p.GetOrigin()
 	release["Label"] = p.GetLabel()
@@ -621,80 +577,36 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 	release["Components"] = strings.Join(p.Components(), " ")
 
-	for path, info := range generatedFiles {
+	for path, info := range indexes.generatedFiles {
 		release["MD5Sum"] += fmt.Sprintf(" %s %8d %s\n", info.MD5, info.Size, path)
 		release["SHA1"] += fmt.Sprintf(" %s %8d %s\n", info.SHA1, info.Size, path)
 		release["SHA256"] += fmt.Sprintf(" %s %8d %s\n", info.SHA256, info.Size, path)
 	}
 
-	var releaseFile *os.File
-	releaseFilename := filepath.Join(tempDir, "Release")
-	releaseFile, err = os.Create(releaseFilename)
+	releaseFile := indexes.ReleaseFile()
+	bufWriter, err := releaseFile.BufWriter()
 	if err != nil {
-		return fmt.Errorf("unable to create temporary Release file: %s", err)
+		return err
 	}
 
-	bufWriter := bufio.NewWriter(releaseFile)
-
 	err = release.WriteTo(bufWriter)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
 
-	err = bufWriter.Flush()
-	if err != nil {
-		return fmt.Errorf("unable to create Release file: %s", err)
-	}
-
-	releaseFile.Close()
-
-	if suffix != "" {
-		renameMap[filepath.Join(basePath, "Release"+suffix)] = filepath.Join(basePath, "Release")
-	}
-
-	err = publishedStorage.PutFile(filepath.Join(basePath, "Release"+suffix), releaseFilename)
-	if err != nil {
-		return fmt.Errorf("unable to publish file: %s", err)
-	}
-
 	// Signing files might output to console, so flush progress writer first
 	if progress != nil {
 		progress.Flush()
 	}
 
-	if signer != nil {
-		err = signer.DetachedSign(releaseFilename, releaseFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		err = signer.ClearSign(releaseFilename, filepath.Join(filepath.Dir(releaseFilename), "InRelease"+suffix))
-		if err != nil {
-			return fmt.Errorf("unable to sign Release file: %s", err)
-		}
-
-		if suffix != "" {
-			renameMap[filepath.Join(basePath, "Release"+suffix+".gpg")] = filepath.Join(basePath, "Release.gpg")
-			renameMap[filepath.Join(basePath, "InRelease"+suffix)] = filepath.Join(basePath, "InRelease")
-		}
-
-		err = publishedStorage.PutFile(filepath.Join(basePath, "Release"+suffix+".gpg"), releaseFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
-		}
-
-		err = publishedStorage.PutFile(filepath.Join(basePath, "InRelease"+suffix),
-			filepath.Join(filepath.Dir(releaseFilename), "InRelease"+suffix))
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
-		}
+	err = releaseFile.Finalize(signer)
+	if err != nil {
+		return err
 	}
 
-	for oldName, newName := range renameMap {
-		err = publishedStorage.RenameFile(oldName, newName)
-		if err != nil {
-			return fmt.Errorf("unable to rename: %s", err)
-		}
+	err = indexes.RenameFiles()
+	if err != nil {
+		return err
 	}
 
 	return nil

deb/publish_test.go

@@ -39,6 +39,9 @@ func (n *NullSigner) SetKey(keyRef string) {
 func (n *NullSigner) SetKeyRing(keyring, secretKeyring string) {
 }
 
+func (n *NullSigner) SetPassphrase(passphrase, passphraseFile string) {
+}
+
 func (n *NullSigner) DetachedSign(source string, destination string) error {
 	return ioutil.WriteFile(destination, []byte{}, 0644)
 }
@@ -90,7 +93,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 		"files:other": s.publishedStorage2}}
 	s.packagePool = files.NewPackagePool(s.root)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -164,6 +167,9 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
 
 	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, ErrorMatches, "duplicate component name: main")
+
+	_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
+	c.Check(err, ErrorMatches, "invalid distribution wheezy/updates, '/' is not allowed")
 }
 
 func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
@@ -264,6 +270,13 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 	c.Check(repo.Distribution, Equals, "precise")
 	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
 
+	s.localRepo.DefaultDistribution = "precise/updates"
+
+	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
+	c.Check(err, IsNil)
+	c.Check(repo.Distribution, Equals, "precise-updates")
+	c.Check(repo.Components(), DeepEquals, []string{"contrib"})
+
 	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
 	c.Check(err, IsNil)
 	c.Check(repo.Distribution, Equals, "squeeze")

deb/query.go

@@ -7,14 +7,22 @@ import (
 	"strings"
 )
 
+// PackageCatalog is abstraction on top of PackageCollection and PackageList
+type PackageCatalog interface {
+	Scan(q PackageQuery) (result *PackageList)
+	Search(dep Dependency, allMatches bool) (searchResults []*Package)
+	SearchSupported() bool
+	SearchByKey(arch, name, version string) (result *PackageList)
+}
+
 // PackageQuery is interface of predicate on Package
 type PackageQuery interface {
 	// Matches calculates match of condition against package
 	Matches(pkg *Package) bool
 	// Fast returns if search strategy is possible for this query
-	Fast() bool
+	Fast(list PackageCatalog) bool
 	// Query performs search on package list
-	Query(list *PackageList) *PackageList
+	Query(list PackageCatalog) *PackageList
 	// String interface
 	String() string
 }
@@ -60,13 +68,13 @@ func (q *OrQuery) Matches(pkg *Package) bool {
 }
 
 // Fast is true only if both parts are fast
-func (q *OrQuery) Fast() bool {
-	return q.L.Fast() && q.R.Fast()
+func (q *OrQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) && q.R.Fast(list)
 }
 
 // Query strategy depends on nodes
-func (q *OrQuery) Query(list *PackageList) (result *PackageList) {
-	if q.Fast() {
+func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
 		result = q.L.Query(list)
 		result.Append(q.R.Query(list))
 	} else {
@@ -86,16 +94,16 @@ func (q *AndQuery) Matches(pkg *Package) bool {
 }
 
 // Fast is true if any of the parts are fast
-func (q *AndQuery) Fast() bool {
-	return q.L.Fast() || q.R.Fast()
+func (q *AndQuery) Fast(list PackageCatalog) bool {
+	return q.L.Fast(list) || q.R.Fast(list)
 }
 
 // Query strategy depends on nodes
-func (q *AndQuery) Query(list *PackageList) (result *PackageList) {
-	if !q.Fast() {
+func (q *AndQuery) Query(list PackageCatalog) (result *PackageList) {
+	if !q.Fast(list) {
 		result = list.Scan(q)
 	} else {
-		if q.L.Fast() {
+		if q.L.Fast(list) {
 			result = q.L.Query(list)
 			result = result.Scan(q.R)
 		} else {
@@ -117,12 +125,12 @@ func (q *NotQuery) Matches(pkg *Package) bool {
 }
 
 // Fast is false
-func (q *NotQuery) Fast() bool {
+func (q *NotQuery) Fast(list PackageCatalog) bool {
 	return false
 }
 
 // Query strategy is scan always
-func (q *NotQuery) Query(list *PackageList) (result *PackageList) {
+func (q *NotQuery) Query(list PackageCatalog) (result *PackageList) {
 	result = list.Scan(q)
 	return
 }
@@ -170,13 +178,13 @@ func (q *FieldQuery) Matches(pkg *Package) bool {
 }
 
 // Query runs iteration through list
-func (q *FieldQuery) Query(list *PackageList) (result *PackageList) {
+func (q *FieldQuery) Query(list PackageCatalog) (result *PackageList) {
 	result = list.Scan(q)
 	return
 }
 
 // Fast depends on the query
-func (q *FieldQuery) Fast() bool {
+func (q *FieldQuery) Fast(list PackageCatalog) bool {
 	return false
 }
 
@@ -215,15 +223,19 @@ func (q *DependencyQuery) Matches(pkg *Package) bool {
 }
 
 // Fast is always true for dependency query
-func (q *DependencyQuery) Fast() bool {
-	return true
+func (q *DependencyQuery) Fast(list PackageCatalog) bool {
+	return list.SearchSupported()
 }
 
 // Query runs PackageList.Search
-func (q *DependencyQuery) Query(list *PackageList) (result *PackageList) {
-	result = NewPackageList()
-	for _, pkg := range list.Search(q.Dep, true) {
-		result.Add(pkg)
+func (q *DependencyQuery) Query(list PackageCatalog) (result *PackageList) {
+	if q.Fast(list) {
+		result = NewPackageList()
+		for _, pkg := range list.Search(q.Dep, true) {
+			result.Add(pkg)
+		}
+	} else {
+		result = list.Scan(q)
 	}
 
 	return
@@ -240,20 +252,13 @@ func (q *PkgQuery) Matches(pkg *Package) bool {
 }
 
 // Fast is always true for package query
-func (q *PkgQuery) Fast() bool {
+func (q *PkgQuery) Fast(list PackageCatalog) bool {
 	return true
 }
 
 // Query looks up specific package
-func (q *PkgQuery) Query(list *PackageList) (result *PackageList) {
-	result = NewPackageList()
-
-	pkg := list.packages["P"+q.Arch+" "+q.Pkg+" "+q.Version]
-	if pkg != nil {
-		result.Add(pkg)
-	}
-
-	return
+func (q *PkgQuery) Query(list PackageCatalog) (result *PackageList) {
+	return list.SearchByKey(q.Arch, q.Pkg, q.Version)
 }
 
 // String interface

deb/reflist.go

@@ -84,6 +84,14 @@ func (l *PackageRefList) ForEach(handler func([]byte) error) error {
 	return err
 }
 
+// Has checks whether package is part of reflist
+func (l *PackageRefList) Has(p *Package) bool {
+	key := p.Key("")
+
+	i := sort.Search(len(l.Refs), func(j int) bool { return bytes.Compare(l.Refs[j], key) >= 0 })
+	return i < len(l.Refs) && bytes.Compare(l.Refs[i], key) == 0
+}
+
 // Substract returns all packages in l that are not in r
 func (l *PackageRefList) Substract(r *PackageRefList) *PackageRefList {
 	result := &PackageRefList{Refs: make([][]byte, 0, 128)}

deb/reflist_test.go

@@ -128,6 +128,19 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestHas(c *C) {
+	s.list.Add(s.p1)
+	s.list.Add(s.p3)
+	s.list.Add(s.p5)
+	reflist := NewPackageRefListFromPackageList(s.list)
+
+	c.Check(reflist.Has(s.p1), Equals, true)
+	c.Check(reflist.Has(s.p3), Equals, true)
+	c.Check(reflist.Has(s.p5), Equals, true)
+	c.Check(reflist.Has(s.p2), Equals, true)
+	c.Check(reflist.Has(s.p6), Equals, false)
+}
+
 func (s *PackageRefListSuite) TestSubstract(c *C) {
 	r1 := []byte("r1")
 	r2 := []byte("r2")

deb/remote.go

@@ -16,9 +16,16 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 )
 
+// RemoteRepo statuses
+const (
+	MirrorIdle = iota
+	MirrorUpdating
+)
+
 // RemoteRepo represents remote (fetchable) Debian repository.
 //
 // Repostitory could be filtered when fetching by components, architectures
@@ -37,6 +44,8 @@ type RemoteRepo struct {
 	Architectures []string
 	// Should we download sources?
 	DownloadSources bool
+	// Should we download .udebs?
+	DownloadUdebs bool
 	// Meta-information about repository
 	Meta Stanza
 	// Last update date
@@ -47,15 +56,23 @@ type RemoteRepo struct {
 	Filter string
 	// FilterWithDeps to include dependencies from filter query
 	FilterWithDeps bool
+	// Status marks state of repository (being updated, no action)
+	Status int
+	// WorkerPID is PID of the process modifying the mirror (if any)
+	WorkerPID int
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
+	// Temporary list of package refs
+	tempPackageRefs *PackageRefList
 	// Parsed archived root
 	archiveRootURL *url.URL
+	// Current list of packages (filled while updating mirror)
+	packageList *PackageList
 }
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
 		UUID:            uuid.New(),
 		Name:            name,
@@ -64,6 +81,7 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		Components:      components,
 		Architectures:   architectures,
 		DownloadSources: downloadSources,
+		DownloadUdebs:   downloadUdebs,
 	}
 
 	err := result.prepare()
@@ -80,6 +98,9 @@ func NewRemoteRepo(name string, archiveRoot string, distribution string, compone
 		if len(result.Components) > 0 {
 			return nil, fmt.Errorf("components aren't supported for flat repos")
 		}
+		if result.DownloadUdebs {
+			return nil, fmt.Errorf("debian-installer udebs aren't supported for flat repos")
+		}
 		result.Components = nil
 	}
 
@@ -102,7 +123,10 @@ func (repo *RemoteRepo) prepare() error {
 func (repo *RemoteRepo) String() string {
 	srcFlag := ""
 	if repo.DownloadSources {
-		srcFlag = " [src]"
+		srcFlag += " [src]"
+	}
+	if repo.DownloadUdebs {
+		srcFlag += " [udeb]"
 	}
 	distribution := repo.Distribution
 	if distribution == "" {
@@ -131,6 +155,37 @@ func (repo *RemoteRepo) RefList() *PackageRefList {
 	return repo.packageRefs
 }
 
+// MarkAsUpdating puts current PID and sets status to updating
+func (repo *RemoteRepo) MarkAsUpdating() {
+	repo.Status = MirrorUpdating
+	repo.WorkerPID = os.Getpid()
+}
+
+// MarkAsIdle clears updating flag
+func (repo *RemoteRepo) MarkAsIdle() {
+	repo.Status = MirrorIdle
+	repo.WorkerPID = 0
+}
+
+// CheckLock returns error if mirror is being updated by another process
+func (repo *RemoteRepo) CheckLock() error {
+	if repo.Status == MirrorIdle || repo.WorkerPID == 0 {
+		return nil
+	}
+
+	p, err := os.FindProcess(repo.WorkerPID)
+	if err != nil {
+		return nil
+	}
+
+	err = p.Signal(syscall.Signal(0))
+	if err == nil {
+		return fmt.Errorf("mirror is locked by update operation, PID %d", repo.WorkerPID)
+	}
+
+	return nil
+}
+
 // ReleaseURL returns URL to Release* files in repo root
 func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
 	var path *url.URL
@@ -169,6 +224,13 @@ func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
 	return repo.archiveRootURL.ResolveReference(path)
 }
 
+// UdebURL returns URL of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
+	path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
+	return repo.archiveRootURL.ResolveReference(path)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -323,12 +385,13 @@ ok:
 	return nil
 }
 
-// Download downloads all repo files
-func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
-	packagePool aptly.PackagePool, ignoreMismatch bool, dependencyOptions int, filterQuery PackageQuery) error {
-	list := NewPackageList()
-
-	progress.Printf("Downloading & parsing package files...\n")
+// DownloadPackageIndexes downloads & parses package index files
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+	ignoreMismatch bool) error {
+	if repo.packageList != nil {
+		panic("packageList != nil")
+	}
+	repo.packageList = NewPackageList()
 
 	// Download and parse all Packages & Source files
 	packagesURLs := [][]string{}
@@ -342,6 +405,9 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
 				packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
+				if repo.DownloadUdebs {
+					packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
+				}
 			}
 			if repo.DownloadSources {
 				packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
@@ -378,13 +444,15 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 
 			if kind == "binary" {
 				p = NewPackageFromControlFile(stanza)
+			} else if kind == "udeb" {
+				p = NewUdebPackageFromControlFile(stanza)
 			} else if kind == "source" {
 				p, err = NewSourcePackageFromControlFile(stanza)
 				if err != nil {
 					return err
 				}
 			}
-			err = list.Add(p)
+			err = repo.packageList.Add(p)
 			if err != nil {
 				return err
 			}
@@ -398,33 +466,30 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 		progress.ShutdownBar()
 	}
 
-	var err error
+	return nil
+}
 
-	if repo.Filter != "" {
-		progress.Printf("Applying filter...\n")
+// ApplyFilter applies filtering to already built PackageList
+func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
+	repo.packageList.PrepareIndex()
 
-		list.PrepareIndex()
+	emptyList := NewPackageList()
+	emptyList.PrepareIndex()
 
-		emptyList := NewPackageList()
-		emptyList.PrepareIndex()
-
-		origPackages := list.Len()
-		list, err = list.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
-		if err != nil {
-			return err
-		}
-
-		progress.Printf("Packages filtered: %d -> %d.\n", origPackages, list.Len())
+	oldLen = repo.packageList.Len()
+	repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
+	if repo.packageList != nil {
+		newLen = repo.packageList.Len()
 	}
+	return
+}
 
-	progress.Printf("Building download queue...\n")
+// BuildDownloadQueue builds queue, discards current PackageList
+func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
+	queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
+	seen := make(map[string]struct{}, repo.packageList.Len())
 
-	// Build download queue
-	queued := make(map[string]PackageDownloadTask, list.Len())
-	count := 0
-	downloadSize := int64(0)
-
-	err = list.ForEach(func(p *Package) error {
+	err = repo.packageList.ForEach(func(p *Package) error {
 		list, err2 := p.DownloadList(packagePool)
 		if err2 != nil {
 			return err2
@@ -433,58 +498,31 @@ func (repo *RemoteRepo) Download(progress aptly.Progress, d aptly.Downloader, co
 
 		for _, task := range list {
 			key := task.RepoURI + "-" + task.DestinationPath
-			_, found := queued[key]
+			_, found := seen[key]
 			if !found {
-				count++
+				queue = append(queue, task)
 				downloadSize += task.Checksums.Size
-				queued[key] = task
+				seen[key] = struct{}{}
 			}
 		}
 
 		return nil
 	})
 	if err != nil {
-		return fmt.Errorf("unable to build download queue: %s", err)
+		return
 	}
 
-	repo.packageRefs = NewPackageRefListFromPackageList(list)
+	repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
 	// free up package list, we don't need it after this point
-	list = nil
+	repo.packageList = nil
 
-	progress.Printf("Download queue: %d items (%s)\n", count, utils.HumanBytes(downloadSize))
-
-	progress.InitBar(downloadSize, true)
-
-	// Download all package files
-	ch := make(chan error, len(queued))
-
-	for _, task := range queued {
-		d.DownloadWithChecksum(repo.PackageURL(task.RepoURI).String(), task.DestinationPath, ch, task.Checksums, ignoreMismatch)
-	}
-
-	// We don't need queued after this point
-	queued = nil
-
-	// Wait for all downloads to finish
-	errors := make([]string, 0)
-
-	for count > 0 {
-		err = <-ch
-		if err != nil {
-			errors = append(errors, err.Error())
-		}
-		count--
-	}
-
-	progress.ShutdownBar()
-
-	if len(errors) > 0 {
-		return fmt.Errorf("download errors:\n  %s\n", strings.Join(errors, "\n  "))
-	}
+	return
+}
 
+// FinalizeDownload swaps for final value of package refs
+func (repo *RemoteRepo) FinalizeDownload() {
 	repo.LastDownloadDate = time.Now()
-
-	return nil
+	repo.packageRefs = repo.tempPackageRefs
 }
 
 // Encode does msgpack encoding of RemoteRepo

deb/remote_test.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	. "launchpad.net/gocheck"
 	"os"
+	"sort"
 )
 
 type NullVerifier struct {
@@ -79,8 +80,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.OpenDB(c.MkDir())
@@ -96,7 +97,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(err, ErrorMatches, ".*hexadecimal escape in host.*")
 }
 
@@ -106,11 +107,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -119,8 +120,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./")
 
 	s.repo.DownloadSources = true
+	s.repo.DownloadUdebs = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -151,6 +153,10 @@ func (s *RemoteRepoSuite) TestBinaryURL(c *C) {
 	c.Assert(s.repo.BinaryURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/binary-amd64/Packages")
 }
 
+func (s *RemoteRepoSuite) TestUdebURL(c *C) {
+	c.Assert(s.repo.UdebURL("main", "amd64").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/debian-installer/binary-amd64/Packages")
+}
+
 func (s *RemoteRepoSuite) TestSourcesURL(c *C) {
 	c.Assert(s.repo.SourcesURL("main").String(), Equals, "http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources")
 }
@@ -209,13 +215,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -249,20 +255,22 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
-	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -279,32 +287,35 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.bz2", errors.New("HTTP 404"))
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", errors.New("HTTP 404"))
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	s.downloader.AnyExpectResponse("http://mirror.yandex.ru/debian/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
-	err = s.repo.Download(s.progress, s.downloader, s.collectionFactory, s.packagePool, false, 0, nil)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.repo.FinalizeDownload()
 	c.Assert(s.repo.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
-
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -314,23 +325,25 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Packages.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Packages", examplePackagesFile)
-	downloader.ExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(3))
+	c.Check(queue, HasLen, 1)
+	c.Check(queue[0].RepoURI, Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 }
 
@@ -345,35 +358,39 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.bz2", errors.New("HTTP 404"))
 	downloader.ExpectError("http://repos.express42.com/virool/precise/Sources.gz", errors.New("HTTP 404"))
 	downloader.ExpectResponse("http://repos.express42.com/virool/precise/Sources", exampleSourcesFile)
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb", "xyz")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc", "abc")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz", "abcd")
-	downloader.AnyExpectResponse("http://repos.express42.com/virool/precise/pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz", "abcde")
 
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.Download(s.progress, downloader, s.collectionFactory, s.packagePool, false, 0, nil)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, false)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
+
+	queue, size, err := s.flat.BuildDownloadQueue(s.packagePool)
+	c.Check(size, Equals, int64(15))
+	c.Check(queue, HasLen, 4)
+
+	q := make([]string, 4)
+	for i := range q {
+		q[i] = queue[i].RepoURI
+	}
+	sort.Strings(q)
+	c.Check(q[3], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+	c.Check(q[1], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.dsc")
+	c.Check(q[2], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0.orig.tar.gz")
+	c.Check(q[0], Equals, "pool/main/a/access-modifier-checker/access-modifier-checker_1.0-4.debian.tar.gz")
+
+	s.flat.FinalizeDownload()
 	c.Assert(s.flat.packageRefs, NotNil)
 
 	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[0])
 	c.Assert(err, IsNil)
 
-	result, err := pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "amanda-client")
 
 	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.flat.packageRefs.Refs[1])
 	c.Assert(err, IsNil)
 
-	result, err = pkg.VerifyFiles(s.packagePool)
-	c.Check(result, Equals, true)
-	c.Check(err, IsNil)
-
 	c.Check(pkg.Name, Equals, "access-modifier-checker")
 }
 
@@ -399,7 +416,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	r, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -417,7 +434,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	r, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err = s.collection.ByUUID(repo.UUID)
@@ -426,7 +443,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -447,7 +464,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -469,10 +486,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot_test.go

@@ -15,7 +15,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -108,11 +108,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -192,7 +192,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot{})
 }

http/download.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"github.com/smira/aptly/aptly"
 	"github.com/smira/aptly/utils"
+	"github.com/smira/go-ftp-protocol/protocol"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -47,11 +48,12 @@ type downloadTask struct {
 func NewDownloader(threads int, downLimit int64, progress aptly.Progress) aptly.Downloader {
 	transport := *http.DefaultTransport.(*http.Transport)
 	transport.DisableCompression = true
+	transport.RegisterProtocol("ftp", &protocol.FTPRoundTripper{})
 
 	downloader := &downloaderImpl{
 		queue:    make(chan *downloadTask, 1000),
-		stop:     make(chan struct{}),
-		stopped:  make(chan struct{}),
+		stop:     make(chan struct{}, threads),
+		stopped:  make(chan struct{}, threads),
 		pause:    make(chan struct{}),
 		unpause:  make(chan struct{}),
 		threads:  threads,
@@ -86,6 +88,13 @@ func (downloader *downloaderImpl) Shutdown() {
 	}
 }
 
+// Abort stops downloader but doesn't wait for downloader to stop
+func (downloader *downloaderImpl) Abort() {
+	for i := 0; i < downloader.threads; i++ {
+		downloader.stop <- struct{}{}
+	}
+}
+
 // Pause pauses task processing
 func (downloader *downloaderImpl) Pause() {
 	for i := 0; i < downloader.threads; i++ {
@@ -122,10 +131,12 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	resp, err := downloader.client.Get(task.url)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
-	defer resp.Body.Close()
+	if resp.Body != nil {
+		defer resp.Body.Close()
+	}
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
 		task.result <- fmt.Errorf("HTTP code %d while fetching %s", resp.StatusCode, task.url)
@@ -134,7 +145,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	err = os.MkdirAll(filepath.Dir(task.destination), 0755)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 
@@ -142,7 +153,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 
 	outfile, err := os.Create(temppath)
 	if err != nil {
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 	defer outfile.Close()
@@ -159,7 +170,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	_, err = io.Copy(w, resp.Body)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 
@@ -190,7 +201,7 @@ func (downloader *downloaderImpl) handleTask(task *downloadTask) {
 	err = os.Rename(temppath, task.destination)
 	if err != nil {
 		os.Remove(temppath)
-		task.result <- err
+		task.result <- fmt.Errorf("%s: %s", task.url, err)
 		return
 	}
 

http/fake.go

@@ -123,6 +123,10 @@ func (f *FakeDownloader) Download(url string, filename string, result chan<- err
 func (f *FakeDownloader) Shutdown() {
 }
 
+// Abort does nothing
+func (f *FakeDownloader) Abort() {
+}
+
 // Pause does nothing
 func (f *FakeDownloader) Pause() {
 }

http/http.go

@@ -1,2 +1,2 @@
-// Package http provides all HTTP-related operations
+// Package http provides all HTTP (and FTP)-related operations
 package http

main.go

@@ -1,38 +1,10 @@
 package main
 
 import (
-	"fmt"
 	"github.com/smira/aptly/cmd"
 	"os"
 )
 
 func main() {
-	defer func() {
-		if r := recover(); r != nil {
-			fatal, ok := r.(*cmd.FatalError)
-			if !ok {
-				panic(r)
-			}
-			fmt.Println("ERROR:", fatal.Message)
-			os.Exit(fatal.ReturnCode)
-		}
-	}()
-
-	command := cmd.RootCommand()
-
-	flags, args, err := command.ParseFlags(os.Args[1:])
-	if err != nil {
-		cmd.Fatal(err)
-	}
-
-	err = cmd.InitContext(flags)
-	if err != nil {
-		cmd.Fatal(err)
-	}
-	defer cmd.ShutdownContext()
-
-	err = command.Dispatch(args)
-	if err != nil {
-		cmd.Fatal(err)
-	}
+	os.Exit(cmd.Run(cmd.RootCommand(), os.Args[1:], true))
 }

man/aptly.1

@@ -1,7 +1,7 @@
 .\" generated with Ronn/v0.7.3
 .\" http://github.com/rtomayko/ronn/tree/0.7.3
 .
-.TH "APTLY" "1" "August 2014" "" ""
+.TH "APTLY" "1" "October 2014" "" ""
 .
 .SH "NAME"
 \fBaptly\fR \- Debian repository management tool
@@ -22,10 +22,10 @@ aptly has integrated help that matches contents of this manual page, to get help
 aptly is a tool to create partial and full mirrors of remote repositories, manage local repositories, filter them, merge, upgrade individual packages, take snapshots and publish them back as Debian repositories\.
 .
 .P
-aptly\(cqs goal is to establish repeatability and controlled changes in a package\-centric environment\. aptly allows to fix a set of packages in a repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\.
+aptly\(cqs goal is to establish repeatability and controlled changes in a package\-centric environment\. aptly allows one to fix a set of packages in a repository, so that package installation and upgrade becomes deterministic\. At the same time aptly allows one to perform controlled, fine\-grained changes in repository contents to transition your package environment to new version\.
 .
 .SH "CONFIGURATION"
-aptly looks for configuration file in \fB/etc/aptly\.conf\fR and \fB~/\.aptly\.conf\fR, if no config file found, new one is created\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\.
+aptly looks for configuration file first in \fB~/\.aptly\.conf\fR then in \fB/etc/aptly\.conf\fR and, if no config file found, new one is created in home directory\. If \fB\-config=\fR flag is specified, aptly would use config file at specified location\. Also aptly needs root directory for database, package and published repository storage\. If not specified, directory defaults to \fB~/\.aptly\fR, it will be created if missing\.
 .
 .P
 Configuration file is stored in JSON format (default values shown below):
@@ -55,7 +55,10 @@ Configuration file is stored in JSON format (default values shown below):
       "awsAccessKeyID": ""
       "awsSecretAccessKey": "",
       "prefix": "",
-      "acl": "public\-read"
+      "acl": "public\-read",
+      "storageClass": "",
+      "encryptionMethod": "",
+      "plusWorkaround": false
     }
 }
 .
@@ -115,7 +118,7 @@ if enabled, all mirrors created would have flag set to download source packages;
 specifies paramaters for short PPA url expansion, if left blank they default to output of \fBlsb_release\fR command
 .
 .TP
-\fBS3PublisEndpoints\fR
+\fBS3PublishEndpoints\fR
 configuration of Amazon S3 publishing endpoints (see below)
 .
 .SH "S3 PUBLISHING ENDPOINTS"
@@ -141,6 +144,18 @@ bucket name
 \fBawsAccessKeyID\fR, \fBawsSecretAccessKey\fR
 (optional) Amazon credentials to access S3 bucket\. If not supplied, environment variables \fBAWS_ACCESS_KEY_ID\fR and \fBAWS_SECRET_ACCESS_KEY\fR are used\.
 .
+.TP
+\fBstorageClass\fR
+(optional) Amazon S3 storage class, defaults to \fBSTANDARD\fR\. Other values available: \fBREDUCED_REDUNDANCY\fR (lower price, lower redundancy)
+.
+.TP
+\fBencryptionMethod\fR
+(optional) server\-side encryption method, defaults to none\. Currently the only available encryption method is \fBAES256\fR
+.
+.TP
+\fBplusWorkaround\fR
+(optional) workaround misbehavior in apt and Amazon S3 for files with \fB+\fR in filename by creating two copies of package files with \fB+\fR in filename: one original and another one with spaces instead of plus signs With \fBplusWorkaround\fR enabled, package files with plus sign would be stored twice\. aptly might not cleanup files with spaces when published repository is dropped or updated (switched) to new version of repository (snapshot)\.
+.
 .P
 In order to publish to S3, specify endpoint as \fBs3:endpoint\-name:\fR before publishing prefix on the command line, e\.g\.:
 .
@@ -274,7 +289,7 @@ when processing dependencies, follow Suggests
 \fBaptly\fR \fBmirror\fR \fBcreate\fR \fIname\fR \fIarchive url\fR \fIdistribution\fR [\fIcomponent1\fR \|\.\|\.\|\.]
 .
 .P
-Creates mirror \fIname\fR of remote repository, aptly supports both regular and flat Debian repositories exported via HTTP\. aptly would try download Release file from remote repository and verify its\(cq signature\. Command line format resembles apt utlitily sources\.list(5)\.
+Creates mirror \fIname\fR of remote repository, aptly supports both regular and flat Debian repositories exported via HTTP and FTP\. aptly would try download Release file from remote repository and verify its\(cq signature\. Command line format resembles apt utlitily sources\.list(5)\.
 .
 .P
 PPA urls could specified in short format:
@@ -311,6 +326,10 @@ gpg keyring to use when verifying Release file (could be specified multiple time
 \-\fBwith\-sources\fR=false
 download source packages in addition to binary packages
 .
+.TP
+\-\fBwith\-udebs\fR=false
+download \.udeb packages (Debian installer support)
+.
 .SH "LIST MIRRORS"
 \fBaptly\fR \fBmirror\fR \fBlist\fR
 .
@@ -388,6 +407,10 @@ Options:
 limit download speed (kbytes/sec)
 .
 .TP
+\-\fBforce\fR=false
+force update mirror even if it is locked by another process
+.
+.TP
 \-\fBignore\-checksums\fR=false
 ignore checksum mismatches while downloading package files and metadata
 .
@@ -411,11 +434,11 @@ Example:
 .P
 $ aptly mirror rename wheezy\-min wheezy\-main
 .
-.SH "EDIT PROPERTIES OF MIRORR"
+.SH "EDIT MIRROR SETTINGS"
 \fBaptly\fR \fBmirror\fR \fBedit\fR \fIname\fR
 .
 .P
-Command edit allows to change settings of mirror: filters\.
+Command edit allows one to change settings of mirror: filters, list of architectures\.
 .
 .P
 Example:
@@ -434,11 +457,45 @@ filter packages in mirror
 \-\fBfilter\-with\-deps\fR=false
 when filtering, include dependencies of matching packages as well
 .
+.TP
+\-\fBwith\-sources\fR=false
+download source packages in addition to binary packages
+.
+.TP
+\-\fBwith\-udebs\fR=false
+download \.udeb packages (Debian installer support)
+.
+.SH "SEARCH MIRROR FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBmirror\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in mirror that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly mirror search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
 .SH "ADD PACKAGES TO LOCAL REPOSITORY"
 \fBaptly\fR \fBrepo\fR \fBadd\fR \fIname\fR
 .
 .P
-Command adds packages to local repository from \.deb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\.
+Command adds packages to local repository from \.deb, \.udeb (binary packages) and \.dsc (source packages) files\. When importing from directory aptly would do recursive scan looking for all files matching \fI\.[u]deb or\fR\.dsc patterns\. Every file discovered would be analyzed to extract metadata, package would then be created and added to the database\. Files would be imported to internal package pool\. For source packages, all required files are added automatically as well\. Extra files for source package should be in the same directory as *\.dsc file\.
 .
 .P
 Example:
@@ -450,6 +507,10 @@ $ aptly repo add testing myapp\-0\.1\.2\.deb incoming/
 Options:
 .
 .TP
+\-\fBforce\-replace\fR=false
+when adding package that conflicts with existing package, remove existing package
+.
+.TP
 \-\fBremove\-files\fR=false
 remove files that have been imported successfully into repository
 .
@@ -526,7 +587,7 @@ force local repo deletion even if used by snapshots
 \fBaptly\fR \fBrepo\fR \fBedit\fR \fIname\fR
 .
 .P
-Command edit allows to change metadata of local repository: comment, default distribution and component\.
+Command edit allows one to change metadata of local repository: comment, default distribution and component\.
 .
 .P
 Example:
@@ -661,6 +722,32 @@ Example:
 .P
 $ aptly repo rename wheezy\-min wheezy\-main
 .
+.SH "SEARCH REPO FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBrepo\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in local repository that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly repo search my\-software \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
 .SH "CREATES SNAPSHOT OF MIRROR (LOCAL REPOSITORY) CONTENTS"
 \fBaptly\fR \fBsnapshot\fR \fBcreate\fR \fIname\fR \fBfrom\fR \fBmirror\fR \fImirror\-name\fR \fB|\fR \fBfrom\fR \fBrepo\fR \fIrepo\-name\fR \fB|\fR \fBempty\fR
 .
@@ -879,6 +966,58 @@ Example:
 .P
 $ aptly snapshot rename wheezy\-min wheezy\-main
 .
+.SH "SEARCH SNAPSHOT FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBsnapshot\fR \fBsearch\fR \fIname\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in snapshot that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot search wheezy\-main \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependencies into search results
+.
+.SH "FILTER PACKAGES IN SNAPSHOT PRODUCING ANOTHER SNAPSHOT"
+\fBaptly\fR \fBsnapshot\fR \fBfilter\fR \fIsource\fR \fIdestination\fR \fIpackage\-query\fR \fB\|\.\|\.\|\.\fR
+.
+.P
+Command filter does filtering in snapshot \fIsource\fR, producing another snapshot \fIdestination\fR\. Packages could be specified simply as \(cqpackage\-name\(cq or as package queries\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly snapshot filter wheezy\-main wheezy\-required \(cqPriorioty (required)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-deps\fR=false
+include dependent packages as well
+.
 .SH "REMOVE PUBLISHED REPOSITORY"
 \fBaptly\fR \fBpublish\fR \fBdrop\fR \fIdistribution\fR [[\fIendpoint\fR:]\fIprefix\fR]
 .
@@ -991,6 +1130,14 @@ label to publish
 origin name to publish
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -1062,6 +1209,14 @@ label to publish
 origin name to publish
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -1121,6 +1276,14 @@ GPG key ID to use when signing the release
 GPG keyring to use (instead of default)
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -1166,6 +1329,14 @@ GPG key ID to use when signing the release
 GPG keyring to use (instead of default)
 .
 .TP
+\-\fBpassphrase\fR=
+GPG passhprase for the key (warning: could be insecure)
+.
+.TP
+\-\fBpassphrase\-file\fR=
+GPG passhprase\-file for the key (warning: could be insecure)
+.
+.TP
 \-\fBsecret\-keyring\fR=
 GPG secret keyring to use (instead of default)
 .
@@ -1173,6 +1344,55 @@ GPG secret keyring to use (instead of default)
 \-\fBskip\-signing\fR=false
 don\(cqt sign Release files with GPG
 .
+.SH "SEARCH FOR PACKAGES MATCHING QUERY"
+\fBaptly\fR \fBpackage\fR \fBsearch\fR \fIpackage\-query\fR
+.
+.P
+Command search displays list of packages in whole DB that match package query
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly package search \(cq$Architecture (i386), Name (% *\-dev)\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.SH "SHOW DETAILS ABOUT PACKAGES MATCING QUERY"
+\fBaptly\fR \fBpackage\fR \fBshow\fR \fIpackage\-query\fR
+.
+.P
+Command shows displays detailed meta\-information about packages matching query\. Information from Debian control file is displayed\. Optionally information about package files and inclusion into mirrors/snapshots/local repos is shown\.
+.
+.P
+Example:
+.
+.IP "" 4
+.
+.nf
+
+$ aptly package show nginx\-light_1\.2\.1\-2\.2+wheezy2_i386\(cq
+.
+.fi
+.
+.IP "" 0
+.
+.P
+Options:
+.
+.TP
+\-\fBwith\-files\fR=false
+display information about files from package pool
+.
+.TP
+\-\fBwith\-references\fR=false
+display information about mirrors, snapshots and local repos referencing this package
+.
 .SH "CLEANUP DB AND PACKAGE POOL"
 \fBaptly\fR \fBdb\fR \fBcleanup\fR
 .

man/aptly.1.ronn.tmpl

@@ -18,8 +18,9 @@ aptly has integrated help that matches contents of this manual page, to get help
 
 ## CONFIGURATION
 
-aptly looks for configuration file in `/etc/aptly.conf` and `~/.aptly.conf`, if no config file
-found, new one is created. If `-config=` flag is specified, aptly would use config file at specified
+aptly looks for configuration file first in `~/.aptly.conf` then
+in `/etc/aptly.conf` and, if no config file found, new one is created in
+home directory. If `-config=` flag is specified, aptly would use config file at specified
 location. Also aptly needs root directory for database, package and published repository storage.
 If not specified, directory defaults to `~/.aptly`, it will be created if missing.
 
@@ -46,7 +47,10 @@ Configuration file is stored in JSON format (default values shown below):
           "awsAccessKeyID": ""
           "awsSecretAccessKey": "",
           "prefix": "",
-          "acl": "public-read"
+          "acl": "public-read",
+          "storageClass": "",
+          "encryptionMethod": "",
+          "plusWorkaround": false
         }
     }
 
@@ -94,7 +98,7 @@ Options:
     specifies paramaters for short PPA url expansion, if left blank they default
     to output of `lsb_release` command
 
-  * `S3PublisEndpoints`:
+  * `S3PublishEndpoints`:
     configuration of Amazon S3 publishing endpoints (see below)
 
 ## S3 PUBLISHING ENDPOINTS
@@ -120,6 +124,20 @@ and associated settings:
      (optional) Amazon credentials to access S3 bucket. If not supplied,
      environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`
      are used.
+   * `storageClass`:
+     (optional) Amazon S3 storage class, defaults to `STANDARD`. Other values
+     available: `REDUCED_REDUNDANCY` (lower price, lower redundancy)
+   * `encryptionMethod`:
+     (optional) server-side encryption method, defaults to none. Currently
+     the only available encryption method is `AES256`
+   * `plusWorkaround`:
+     (optional) workaround misbehavior in apt and Amazon S3
+     for files with `+` in filename by
+     creating two copies of package files with `+` in filename: one original
+     and another one with spaces instead of plus signs
+     With `plusWorkaround` enabled, package files with plus sign
+     would be stored twice. aptly might not cleanup files with spaces when published
+     repository is dropped or updated (switched) to new version of repository (snapshot).
 
 In order to publish to S3, specify endpoint as `s3:endpoint-name:` before
 publishing prefix on the command line, e.g.:
@@ -219,6 +237,8 @@ When specified on command line, query may have to be quoted according to shell r
 
 {{template "command" findCommand . "publish"}}
 
+{{template "command" findCommand . "package"}}
+
 {{template "command" findCommand . "db"}}
 
 {{template "command" findCommand . "serve"}}

s3/public.go

@@ -13,10 +13,13 @@ import (
 
 // PublishedStorage abstract file system with published files (actually hosted on S3)
 type PublishedStorage struct {
-	s3     *s3.S3
-	bucket *s3.Bucket
-	acl    s3.ACL
-	prefix string
+	s3               *s3.S3
+	bucket           *s3.Bucket
+	acl              s3.ACL
+	prefix           string
+	storageClass     string
+	encryptionMethod string
+	plusWorkaround   bool
 }
 
 // Check interface
@@ -25,12 +28,23 @@ var (
 )
 
 // NewPublishedStorageRaw creates published storage from raw aws credentials
-func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix string) (*PublishedStorage, error) {
+func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
 	if defaultACL == "" {
 		defaultACL = "private"
 	}
 
-	result := &PublishedStorage{s3: s3.New(auth, region), acl: s3.ACL(defaultACL), prefix: prefix}
+	if storageClass == "STANDARD" {
+		storageClass = ""
+	}
+
+	result := &PublishedStorage{
+		s3:               s3.New(auth, region),
+		acl:              s3.ACL(defaultACL),
+		prefix:           prefix,
+		storageClass:     storageClass,
+		encryptionMethod: encryptionMethod,
+		plusWorkaround:   plusWorkaround}
 	result.bucket = result.s3.Bucket(bucket)
 
 	return result, nil
@@ -38,7 +52,8 @@ func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL
 
 // NewPublishedStorage creates new instance of PublishedStorage with specified S3 access
 // keys, region and bucket name
-func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix string) (*PublishedStorage, error) {
+func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefix,
+	storageClass, encryptionMethod string, plusWorkaround bool) (*PublishedStorage, error) {
 	auth, err := aws.GetAuth(accessKey, secretKey)
 	if err != nil {
 		return nil, err
@@ -49,7 +64,7 @@ func NewPublishedStorage(accessKey, secretKey, region, bucket, defaultACL, prefi
 		return nil, fmt.Errorf("unknown region: %#v", region)
 	}
 
-	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix)
+	return NewPublishedStorageRaw(auth, awsRegion, bucket, defaultACL, prefix, storageClass, encryptionMethod, plusWorkaround)
 }
 
 // String
@@ -81,10 +96,24 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 		return err
 	}
 
-	err = storage.bucket.PutReader(filepath.Join(storage.prefix, path), source, fi.Size(), "binary/octet-stream", storage.acl)
+	headers := map[string][]string{
+		"Content-Type": {"binary/octet-stream"},
+	}
+	if storage.storageClass != "" {
+		headers["x-amz-storage-class"] = []string{storage.storageClass}
+	}
+	if storage.encryptionMethod != "" {
+		headers["x-amz-server-side-encryption"] = []string{storage.encryptionMethod}
+	}
+
+	err = storage.bucket.PutReaderHeader(filepath.Join(storage.prefix, path), source, fi.Size(), headers, storage.acl)
 	if err != nil {
 		return fmt.Errorf("error uploading %s to %s: %s", sourceFilename, storage, err)
 	}
+
+	if storage.plusWorkaround && strings.Index(path, "+") != -1 {
+		return storage.PutFile(strings.Replace(path, "+", " ", -1), sourceFilename)
+	}
 	return nil
 }
 

s3/public_test.go

@@ -24,10 +24,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 	c.Assert(s.srv, NotNil)
 
 	auth, _ := aws.GetAuth("aa", "bb")
-	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "")
+	s.storage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "", "", "", false)
 	c.Assert(err, IsNil)
 
-	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala")
+	s.prefixedStorage, err = NewPublishedStorageRaw(auth, aws.Region{Name: "test-1", S3Endpoint: s.srv.URL(), S3LocationConstraint: true}, "test", "", "lala", "", "", false)
 	c.Assert(err, IsNil)
 
 	err = s.storage.s3.Bucket("test").PutBucket("private")
@@ -39,7 +39,7 @@ func (s *PublishedStorageSuite) TearDownTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TestNewPublishedStorage(c *C) {
-	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "")
+	stor, err := NewPublishedStorage("aa", "bbb", "", "", "", "", "", "", false)
 	c.Check(stor, IsNil)
 	c.Check(err, ErrorMatches, "unknown region: .*")
 }
@@ -64,6 +64,25 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	c.Check(data, DeepEquals, []byte("welcome to s3!"))
 }
 
+func (s *PublishedStorageSuite) TestPutFilePlusWorkaround(c *C) {
+	s.storage.plusWorkaround = true
+
+	dir := c.MkDir()
+	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("welcome to s3!"), 0644)
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("a/b+c.txt", filepath.Join(dir, "a"))
+	c.Check(err, IsNil)
+
+	data, err := s.storage.bucket.Get("a/b+c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+
+	data, err = s.storage.bucket.Get("a/b c.txt")
+	c.Check(err, IsNil)
+	c.Check(data, DeepEquals, []byte("welcome to s3!"))
+}
+
 func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	paths := []string{"a", "b", "c", "testa", "test/a", "test/b", "lala/a", "lala/b", "lala/c"}
 	for _, path := range paths {

system/lib.py

@@ -155,6 +155,7 @@ class BaseTest(object):
             if not hasattr(command, "__iter__"):
                 params = {
                     'files': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files"),
+                    'udebs': os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "udebs"),
                     'testfiles': os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), self.__class__.__name__),
                 }
                 if self.fixtureWebServer:
@@ -195,17 +196,21 @@ class BaseTest(object):
             self.verify_match(self.get_gold(), self.output, match_prepare=self.outputMatchPrepare)
         except:
             if self.captureResults:
+                if self.outputMatchPrepare is not None:
+                    self.output = self.outputMatchPrepare(self.output)
                 with open(self.get_gold_filename(), "w") as f:
                     f.write(self.output)
             else:
                 raise
 
     def check_cmd_output(self, command, gold_name, match_prepare=None, expected_code=0):
+        output = self.run_cmd(command, expected_code=expected_code)
         try:
-            output = self.run_cmd(command, expected_code=expected_code)
             self.verify_match(self.get_gold(gold_name), output, match_prepare)
         except:
             if self.captureResults:
+                if match_prepare is not None:
+                    output = match_prepare(output)
                 with open(self.get_gold_filename(gold_name), "w") as f:
                     f.write(output)
             else:

system/run.py

@@ -8,6 +8,7 @@ import fnmatch
 import sys
 
 from lib import BaseTest
+from s3_lib import S3Test
 
 try:
     from termcolor import colored
@@ -32,7 +33,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
         for name in dir(testModule):
             o = getattr(testModule, name)
 
-            if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest):
+            if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
+                    o is not S3Test):
                 continue
 
             if filters:

system/s3_lib.py

@@ -0,0 +1,78 @@
+from lib import BaseTest
+import uuid
+import os
+
+try:
+    import boto
+
+    if 'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ:
+        s3_conn = boto.connect_s3()
+    else:
+        s3_conn = None
+except ImportError:
+    s3_conn = None
+
+
+class S3Test(BaseTest):
+    """
+    BaseTest + support for S3
+    """
+
+    def fixture_available(self):
+        return super(S3Test, self).fixture_available() and s3_conn is not None
+
+    def prepare(self):
+        self.bucket_name = "aptly-sys-test-" + str(uuid.uuid4())
+        self.bucket = s3_conn.create_bucket(self.bucket_name)
+        self.configOverride["S3PublishEndpoints"] = {
+            "test1": {
+                "region": "us-east-1",
+                "bucket": self.bucket_name,
+            }
+        }
+
+        super(S3Test, self).prepare()
+
+    def shutdown(self):
+        if hasattr(self, "bucket_name"):
+            if hasattr(self, "bucket"):
+                keys = self.bucket.list()
+                if keys:
+                    self.bucket.delete_keys(keys)
+            s3_conn.delete_bucket(self.bucket_name)
+
+        super(S3Test, self).shutdown()
+
+    def check_path(self, path):
+        if not hasattr(self, "bucket_contents"):
+            self.bucket_contents = [key.name for key in self.bucket.list()]
+
+        if path.startswith("public/"):
+            path = path[7:]
+
+        if path in self.bucket_contents:
+            return True
+
+        if not path.endswith("/"):
+            path = path + "/"
+
+        for item in self.bucket_contents:
+            if item.startswith(path):
+                return True
+
+        return False
+
+    def check_exists(self, path):
+        if not self.check_path(path):
+            raise Exception("path %s doesn't exist" % (path, ))
+
+    def check_not_exists(self, path):
+        if self.check_path(path):
+            raise Exception("path %s exists" % (path, ))
+
+    def read_file(self, path):
+        if path.startswith("public/"):
+            path = path[7:]
+
+        key = self.bucket.get_key(path)
+        return key.get_contents_as_string()

system/t01_version/VersionTest_gold

@@ -1 +1 @@
-aptly version: 0.7.1
+aptly version: 0.8

system/t03_help/MainHelpTest_gold

@@ -4,9 +4,9 @@ upgrade individual packages, take snapshots and publish them
 back as Debian repositories.
 
 aptly's goal is to establish repeatability and controlled changes
-in a package-centric environment. aptly allows to fix a set of packages
+in a package-centric environment. aptly allows one to fix a set of packages
 in a repository, so that package installation and upgrade becomes
-deterministic. At the same time aptly allows to perform controlled,
+deterministic. At the same time aptly allows one to perform controlled,
 fine-grained changes in repository contents to transition your
 package environment to new version.
 

system/t03_help/MainTest_gold

@@ -5,6 +5,7 @@ Commands:
     db          manage aptly's internal database and package pool
     graph       render graph of relationships
     mirror      manage mirrors of remote repositories
+    package     operations on packages
     publish     manage published repositories
     repo        manage local package repositories
     serve       HTTP serve published repositories

system/t03_help/MirrorCreateHelpTest_gold

@@ -1,7 +1,7 @@
 Usage: aptly mirror create <name> <archive url> <distribution> [<component1> ...]
 
 Creates mirror <name> of remote repository, aptly supports both regular and flat Debian repositories exported
-via HTTP. aptly would try download Release file from remote repository and verify its' signature. Command
+via HTTP and FTP. aptly would try download Release file from remote repository and verify its' signature. Command
 line format resembles apt utlitily sources.list(5).
 
 PPA urls could specified in short format:
@@ -24,4 +24,5 @@ Options:
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 

system/t03_help/MirrorCreateTest_gold

@@ -15,4 +15,5 @@ Options:
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 ERROR: unable to parse command

system/t03_help/MirrorHelpTest_gold

@@ -4,9 +4,10 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
-    edit        edit properties of mirorr
+    edit        edit mirror settings
     list        list mirrors
     rename      renames mirror
+    search      search mirror for packages matching query
     show        show details about mirror
     update      update mirror
 

system/t03_help/MirrorTest_gold

@@ -4,9 +4,10 @@ Commands:
 
     create      create new mirror
     drop        delete mirror
-    edit        edit properties of mirorr
+    edit        edit mirror settings
     list        list mirrors
     rename      renames mirror
+    search      search mirror for packages matching query
     show        show details about mirror
     update      update mirror
 

system/t03_help/WrongFlagTest_gold

@@ -16,4 +16,5 @@ Options:
   -ignore-signatures=false: disable verification of Release file signatures
   -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times)
   -with-sources=false: download source packages in addition to binary packages
+  -with-udebs=false: download .udeb packages (Debian installer support)
 ERROR: unable to parse flags

system/t04_mirror/CreateMirror11Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: squeeze
 Components: main, contrib, non-free
 Architectures: amd64, armel, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror13Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main, contrib, non-free
 Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror14Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: ./
 Components: 
 Architectures: 
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror17Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main, contrib, non-free
 Architectures: i386
 Download Sources: yes
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror18Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: maverick
 Components: main
 Architectures: amd64, armel, i386, powerpc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror19Test_mirror_show

@@ -4,17 +4,16 @@ Distribution: wheezy/updates
 Components: main
 Architectures: i386
 Download Sources: yes
+Download .udebs: no
 Last update: never
 
 Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: updates/main updates/contrib updates/non-free
-Date: Tue, 11 Mar 2014 21:11:28 UTC
 Description:  Debian 7.0 Security Updates
 
 Label: Debian-Security
 Origin: Debian
 Suite: stable
-Valid-Until: Fri, 21 Mar 2014 21:11:28 UTC
 Version: 7.0

system/t04_mirror/CreateMirror1Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main, contrib, non-free
 Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror20Test_gold

@@ -1,3 +1,3 @@
 Downloading http://security.debian.org/dists/wheezy/updates/InRelease...
 Downloading http://security.debian.org/dists/wheezy/updates/Release...
-ERROR: unable to fetch mirror: Get http://security.debian.org/dists/wheezy/updates/Release: http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused
+ERROR: unable to fetch mirror: http://security.debian.org/dists/wheezy/updates/Release: Get http://security.debian.org/dists/wheezy/updates/Release: http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused

system/t04_mirror/CreateMirror21Test_mirror_show

@@ -4,10 +4,11 @@ Distribution: ./binary/
 Components: 
 Architectures: 
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:
 Architectures: all
-Date: Wed, 30 Jul 2014 15:23:01 UTC
+Date: Wed, 01 Oct 2014 18:30:34 UTC
 Origin: jenkins-ci.org
 Suite: binary

system/t04_mirror/CreateMirror22Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy/updates
 Components: main
 Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
+Download .udebs: no
 Filter: nginx | Priority (required)
 Filter With Deps: no
 Last update: never
@@ -12,11 +13,9 @@ Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: updates/main updates/contrib updates/non-free
-Date: Sun, 13 Jul 2014 12:12:08 UTC
 Description:  Debian 7.0 Security Updates
 
 Label: Debian-Security
 Origin: Debian
 Suite: stable
-Valid-Until: Wed, 23 Jul 2014 12:12:08 UTC
 Version: 7.0

system/t04_mirror/CreateMirror24Test_gold

@@ -0,0 +1,6 @@
+Downloading http://security.debian.org/dists/wheezy/updates/InRelease...
+gpgv:  RSA key ID 46925553
+gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) <ftpmaster@debian.org>"
+
+Mirror [mirror24]: http://security.debian.org/ wheezy/updates successfully added.
+You can run 'aptly mirror update mirror24' to download repository contents.

system/t04_mirror/CreateMirror25Test_gold

@@ -0,0 +1,4 @@
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release...
+
+Mirror [mirror25]: http://mirror.yandex.ru/debian/ wheezy [udeb] successfully added.
+You can run 'aptly mirror update mirror25' to download repository contents.

system/t04_mirror/CreateMirror25Test_mirror_show

@@ -0,0 +1,20 @@
+Name: mirror25
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main, contrib, non-free
+Architectures: i386
+Download Sources: no
+Download .udebs: yes
+Last update: never
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.6

system/t04_mirror/CreateMirror26Test_gold

@@ -0,0 +1 @@
+ERROR: unable to create mirror: debian-installer udebs aren't supported for flat repos

system/t04_mirror/CreateMirror2Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main
 Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror3Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main, contrib
 Architectures: i386, amd64
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror7Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: wheezy
 Components: main, contrib
 Architectures: i386, amd64
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:

system/t04_mirror/CreateMirror9Test_mirror_show

@@ -4,6 +4,7 @@ Distribution: squeeze-backports
 Components: main, contrib, non-free
 Architectures: amd64, armel, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:
@@ -11,11 +12,9 @@ Architectures: amd64 armel i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel po
 ButAutomaticUpgrades: yes
 Codename: squeeze-backports
 Components: main contrib non-free
-Date: Fri, 07 Feb 2014 02:56:49 UTC
 Description:  Backports for the Squeeze Distribution
 
 Label: Debian Backports
 NotAutomatic: yes
 Origin: Debian Backports
 Suite: squeeze-backports
-Valid-Until: Fri, 14 Feb 2014 02:56:49 UTC

system/t04_mirror/EditMirror1Test_gold

@@ -1 +1 @@
-Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy successfully updated.
+Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy [src] successfully updated.

system/t04_mirror/EditMirror1Test_mirror_show

@@ -3,10 +3,10 @@ Archive Root URL: http://mirror.yandex.ru/debian/
 Distribution: wheezy
 Components: main
 Architectures: i386, amd64
-Download Sources: no
+Download Sources: yes
+Download .udebs: no
 Filter: nginx
 Filter With Deps: yes
-Last update: 2014-06-28 01:23:25 MSK
 Number of packages: 56121
 
 Information from release file:

system/t04_mirror/EditMirror3Test_mirror_show

@@ -4,7 +4,7 @@ Distribution: wheezy
 Components: main
 Architectures: i386, amd64
 Download Sources: no
-Last update: 2014-06-28 01:23:25 MSK
+Download .udebs: no
 Number of packages: 56121
 
 Information from release file:

system/t04_mirror/EditMirror5Test_mirror_show

@@ -4,17 +4,16 @@ Distribution: wheezy/updates
 Components: main
 Architectures: amd64, armel, armhf, i386, ia64, kfreebsd-amd64, kfreebsd-i386, mips, mipsel, powerpc, s390, s390x, sparc
 Download Sources: no
+Download .udebs: no
 Last update: never
 
 Information from release file:
 Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
 Codename: wheezy
 Components: updates/main updates/contrib updates/non-free
-Date: Fri, 25 Jul 2014 03:31:55 UTC
 Description:  Debian 7.0 Security Updates
 
 Label: Debian-Security
 Origin: Debian
 Suite: stable
-Valid-Until: Mon, 04 Aug 2014 03:31:55 UTC
 Version: 7.0

system/t04_mirror/EditMirror6Test_gold

@@ -0,0 +1,2 @@
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release...
+Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy successfully updated.

system/t04_mirror/EditMirror6Test_mirror_show

@@ -0,0 +1,20 @@
+Name: wheezy-main
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main
+Architectures: amd64, s390
+Download Sources: no
+Download .udebs: no
+Number of packages: 56121
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 12 Jul 2014 10:59:25 UTC
+Description:  Debian 7.6 Released 12 July 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.6

system/t04_mirror/EditMirror7Test_gold

@@ -0,0 +1,2 @@
+Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release...
+ERROR: unable to edit: architecture x56 not available in repo [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy

system/t04_mirror/EditMirror8Test_gold

@@ -0,0 +1 @@
+Mirror [wheezy-main]: http://mirror.yandex.ru/debian/ wheezy [udeb] successfully updated.

system/t04_mirror/EditMirror8Test_mirror_show

@@ -0,0 +1,20 @@
+Name: wheezy-main
+Archive Root URL: http://mirror.yandex.ru/debian/
+Distribution: wheezy
+Components: main
+Architectures: i386, amd64
+Download Sources: no
+Download .udebs: yes
+Number of packages: 56121
+
+Information from release file:
+Architectures: amd64 armel armhf i386 ia64 kfreebsd-amd64 kfreebsd-i386 mips mipsel powerpc s390 s390x sparc
+Codename: wheezy
+Components: main contrib non-free
+Date: Sat, 26 Apr 2014 09:27:11 UTC
+Description:  Debian 7.5 Released 26 April 2014
+
+Label: Debian
+Origin: Debian
+Suite: stable
+Version: 7.5