New upstream version 1.4.0+ds1

deb/changes.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -8,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"text/template"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/pgp"
@@ -93,8 +95,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		text = input
 	}
 
-	reader := NewControlFileReader(text)
-	c.Stanza, err = reader.ReadStanza(false)
+	reader := NewControlFileReader(text, false, false)
+	c.Stanza, err = reader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -164,7 +166,7 @@ func (c *Changes) Cleanup() error {
 }
 
 // PackageQuery returns query that every package should match to be included
-func (c *Changes) PackageQuery() (PackageQuery, error) {
+func (c *Changes) PackageQuery() PackageQuery {
 	var archQuery PackageQuery = &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: ""}
 	for _, arch := range c.Architectures {
 		archQuery = &OrQuery{L: &FieldQuery{Field: "$Architecture", Relation: VersionEqual, Value: arch}, R: archQuery}
@@ -215,7 +217,7 @@ func (c *Changes) PackageQuery() (PackageQuery, error) {
 		nameQuery = &OrQuery{L: sourceQuery, R: binaryQuery}
 	}
 
-	return &AndQuery{L: archQuery, R: nameQuery}, nil
+	return &AndQuery{L: archQuery, R: nameQuery}
 }
 
 // GetField implements PackageLike interface
@@ -288,3 +290,143 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 
 	return
 }
+
+// ImportChangesFiles imports referenced files in changes files into local repository
+func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
+	verifier pgp.Verifier, repoTemplateString string, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
+	pool aptly.PackagePool, checksumStorage aptly.ChecksumStorage, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
+
+	var repoTemplate *template.Template
+	repoTemplate, err = template.New("repo").Parse(repoTemplateString)
+	if err != nil {
+		return nil, nil, fmt.Errorf("error parsing -repo template: %s", err)
+	}
+	for _, path := range changesFiles {
+		var changes *Changes
+
+		changes, err = NewChanges(path)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", path, err)
+			continue
+		}
+
+		err = changes.VerifyAndParse(acceptUnsigned, ignoreSignatures, verifier)
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		err = changes.Prepare()
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		repoName := &bytes.Buffer{}
+		err = repoTemplate.Execute(repoName, changes.Stanza)
+		if err != nil {
+			return nil, nil, fmt.Errorf("error applying template to repo: %s", err)
+		}
+
+		if progress != nil {
+			progress.Printf("Loading repository %s for changes file %s...\n", repoName.String(), changes.ChangesName)
+		}
+
+		var repo *LocalRepo
+		repo, err = localRepoCollection.ByName(repoName.String())
+		if err != nil {
+			failedFiles = append(failedFiles, path)
+			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
+			changes.Cleanup()
+			continue
+		}
+
+		currentUploaders := uploaders
+		if repo.Uploaders != nil {
+			currentUploaders = repo.Uploaders
+			for i := range currentUploaders.Rules {
+				currentUploaders.Rules[i].CompiledCondition, err = parseQuery(currentUploaders.Rules[i].Condition)
+				if err != nil {
+					return nil, nil, fmt.Errorf("error parsing query %s: %s", currentUploaders.Rules[i].Condition, err)
+				}
+			}
+		}
+
+		if currentUploaders != nil {
+			if err = currentUploaders.IsAllowed(changes); err != nil {
+				failedFiles = append(failedFiles, path)
+				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
+					changes.ChangesName, changes.SignatureKeys, err)
+				changes.Cleanup()
+				continue
+			}
+		}
+
+		err = localRepoCollection.LoadComplete(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
+		}
+
+		var list *PackageList
+		list, err = NewPackageListFromRefList(repo.RefList(), packageCollection, progress)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to load packages: %s", err)
+		}
+
+		packageFiles, otherFiles, _ := CollectPackageFiles([]string{changes.TempDir}, reporter)
+
+		restriction := changes.PackageQuery()
+		var processedFiles2, failedFiles2 []string
+
+		processedFiles2, failedFiles2, err = ImportPackageFiles(list, packageFiles, forceReplace, verifier, pool,
+			packageCollection, reporter, restriction, checksumStorage)
+
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
+		}
+
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
+
+		err = localRepoCollection.Update(repo)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to save: %s", err)
+		}
+
+		err = changes.Cleanup()
+		if err != nil {
+			return nil, nil, err
+		}
+
+		for _, file := range failedFiles2 {
+			failedFiles = append(failedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range processedFiles2 {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		for _, file := range otherFiles {
+			processedFiles = append(processedFiles, filepath.Join(changes.BasePath, filepath.Base(file)))
+		}
+
+		processedFiles = append(processedFiles, path)
+	}
+
+	if !noRemoveFiles {
+		processedFiles = utils.StrSliceDeduplicate(processedFiles)
+
+		for _, file := range processedFiles {
+			err = os.Remove(file)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to remove file: %s", err)
+			}
+		}
+	}
+
+	return processedFiles, failedFiles, nil
+}

deb/changes_test.go

@@ -4,24 +4,52 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/console"
+	"github.com/aptly-dev/aptly/database"
+	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/utils"
+
 	. "gopkg.in/check.v1"
 )
 
 type ChangesSuite struct {
-	Dir, Path string
+	Dir, Path           string
+	Reporter            aptly.ResultReporter
+	db                  database.Storage
+	localRepoCollection *LocalRepoCollection
+	packageCollection   *PackageCollection
+	packagePool         aptly.PackagePool
+	checksumStorage     aptly.ChecksumStorage
+	progress            aptly.Progress
 }
 
 var _ = Suite(&ChangesSuite{})
 
 func (s *ChangesSuite) SetUpTest(c *C) {
+	s.Reporter = &aptly.RecordingResultReporter{
+		Warnings:     []string{},
+		AddedLines:   []string{},
+		RemovedLines: []string{},
+	}
 	s.Dir = c.MkDir()
 	s.Path = filepath.Join(s.Dir, "calamares.changes")
-
-	f, err := os.Create(s.Path)
+	err := utils.CopyFile("testdata/changes/calamares.changes", s.Path)
 	c.Assert(err, IsNil)
 
-	f.WriteString(changesFile)
-	f.Close()
+	s.db, _ = database.NewOpenDB(c.MkDir())
+	s.localRepoCollection = NewLocalRepoCollection(s.db)
+	s.packageCollection = NewPackageCollection(s.db)
+
+	s.checksumStorage = files.NewMockChecksumStorage()
+	s.packagePool = files.NewPackagePool(s.Dir, false)
+	s.progress = console.NewProgress()
+	s.progress.Start()
+}
+
+func (s *ChangesSuite) TearDownTest(c *C) {
+	s.progress.Shutdown()
+	s.db.Close()
 }
 
 func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -44,6 +72,73 @@ func (s *ChangesSuite) TestParseAndVerify(c *C) {
 	c.Check(changes.Binary, DeepEquals, []string{"calamares", "calamares-dbg"})
 }
 
+func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
+	changesFiles, failedFiles := CollectChangesFiles([]string{"testdata/changes"}, s.Reporter)
+
+	c.Check(failedFiles, HasLen, 0)
+	c.Check(changesFiles, DeepEquals, []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	})
+}
+
+func (s *ChangesSuite) TestImportChangesFiles(c *C) {
+	repo := NewLocalRepo("test", "Test Comment")
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
+
+	origFailedFiles := []string{
+		"testdata/changes/calamares.changes",
+		"testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes",
+		"testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes",
+		"testdata/changes/hardlink_0.2.0_i386.deb",
+	}
+	origProcessedFiles := []string{
+		"testdata/changes/hardlink_0.2.1.dsc",
+		"testdata/changes/hardlink_0.2.1.tar.gz",
+		"testdata/changes/hardlink_0.2.1_amd64.deb",
+		"testdata/changes/hardlink_0.2.1_amd64.buildinfo",
+		"testdata/changes/hardlink_0.2.1_amd64.changes",
+	}
+
+	var expectedProcessedFiles, expectedFailedFiles []string
+
+	for _, path := range origFailedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedFailedFiles = append(expectedFailedFiles, filename)
+	}
+
+	for _, path := range origProcessedFiles {
+		filename := filepath.Join(s.Dir, filepath.Base(path))
+		utils.CopyFile(path, filename)
+		expectedProcessedFiles = append(expectedProcessedFiles, filename)
+	}
+
+	changesFiles, failedFiles := CollectChangesFiles([]string{s.Dir}, s.Reporter)
+	c.Check(failedFiles, HasLen, 0)
+
+	processedFiles, failedFiles, err := ImportChangesFiles(
+		append(changesFiles, "testdata/changes/notexistent.changes"),
+		s.Reporter, true, true, false, false, &NullVerifier{},
+		"test", s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, s.checksumStorage,
+		nil, nil)
+	c.Assert(err, IsNil)
+	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
+	c.Check(processedFiles, DeepEquals, expectedProcessedFiles)
+}
+
+func (s *ChangesSuite) TestPrepare(c *C) {
+	changes, err := NewChanges("testdata/changes/hardlink_0.2.1_amd64.changes")
+	c.Assert(err, IsNil)
+	err = changes.Prepare()
+	c.Assert(err, IsNil)
+
+	_, err = os.Stat(filepath.Join(changes.TempDir, "hardlink_0.2.1_amd64.changes"))
+	c.Check(err, IsNil)
+}
+
 func (s *ChangesSuite) TestPackageQuery(c *C) {
 	changes, err := NewChanges(s.Path)
 	c.Assert(err, IsNil)
@@ -51,42 +146,7 @@ func (s *ChangesSuite) TestPackageQuery(c *C) {
 	err = changes.VerifyAndParse(true, true, &NullVerifier{})
 	c.Check(err, IsNil)
 
-	q, err := changes.PackageQuery()
-	c.Check(err, IsNil)
-
+	q := changes.PackageQuery()
 	c.Check(q.String(), Equals,
 		"(($Architecture (= amd64)) | (($Architecture (= source)) | ($Architecture (= )))), ((($PackageType (= source)), (Name (= calamares))) | ((!($PackageType (= source))), (((Name (= calamares-dbg)) | (Name (= calamares))) | ((Source (= calamares)), ((Name (= calamares-dbg-dbgsym)) | (Name (= calamares-dbgsym)))))))")
 }
-
-var changesFile = `Format: 1.8
-Date: Thu, 27 Nov 2014 13:24:53 +0000
-Source: calamares
-Binary: calamares calamares-dbg
-Architecture: source amd64
-Version: 0+git20141127.99
-Distribution: sid
-Urgency: medium
-Maintainer: Rohan Garg <rohan@kde.org>
-Changed-By: Rohan <rohan@kde.org>
-Description:
- calamares  - distribution-independent installer framework
- calamares-dbg - distribution-independent installer framework -- debug symbols
-Changes:
- calamares (0+git20141127.99) sid; urgency=medium
- .
-   * Update from git
-Checksums-Sha1:
- 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
- 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
- d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
- a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Checksums-Sha256:
- 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
- 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
- 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
- 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
-Files:
- 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
- 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
- 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
- e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb`

deb/contents.go

@@ -7,7 +7,7 @@ import (
 	"io"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 )
 
 // ContentsIndex calculates mapping from files to packages, with sorting and aggregation

deb/deb.go

@@ -11,12 +11,12 @@ import (
 	"strings"
 
 	"github.com/h2non/filetype/matchers"
-	"github.com/mkrautz/goar"
+	ar "github.com/mkrautz/goar"
 	"github.com/pkg/errors"
 
 	"github.com/aptly-dev/aptly/pgp"
+	"github.com/kjk/lzma"
 	"github.com/smira/go-xz"
-	"github.com/smira/lzma"
 )
 
 // Source kinds
@@ -26,6 +26,8 @@ const (
 	SourceRemoteRepo = "repo"
 )
 
+type parseQuery func(string) (PackageQuery, error)
+
 // GetControlFileFromDeb reads control file from deb package
 func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	file, err := os.Open(packageFile)
@@ -87,8 +89,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
-					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza(false)
+					reader := NewControlFileReader(untar, false, false)
+					stanza, err := reader.ReadStanza()
 					if err != nil {
 						return nil, err
 					}
@@ -127,8 +129,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		text = file
 	}
 
-	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza(false)
+	reader := NewControlFileReader(text, false, false)
+	stanza, err := reader.ReadStanza()
 	if err != nil {
 		return nil, err
 	}

deb/format.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"errors"
 	"io"
+	"sort"
 	"strings"
 	"unicode"
 )
@@ -88,6 +89,9 @@ var (
 		"Directory",
 		"Files",
 	}
+	canonicalOrderInstaller = []string{
+		"",
+	}
 )
 
 // Copy returns copy of Stanza
@@ -101,6 +105,9 @@ func (s Stanza) Copy() (result Stanza) {
 
 func isMultilineField(field string, isRelease bool) bool {
 	switch field {
+	// file without a section
+	case "":
+		return true
 	case "Description":
 		return true
 	case "Files":
@@ -127,26 +134,33 @@ func isMultilineField(field string, isRelease bool) bool {
 	return false
 }
 
-// Write single field from Stanza to writer
+// Write single field from Stanza to writer.
+//
+//nolint: interfacer
 func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
 	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
-		if !strings.HasSuffix(value, "\n") {
+		if field != "" && !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
 
-		if field != "Description" {
+		if field != "Description" && field != "" {
 			value = "\n" + value
 		}
-		_, err = w.WriteString(field + ":" + value)
+
+		if field != "" {
+			_, err = w.WriteString(field + ":" + value)
+		} else {
+			_, err = w.WriteString(value)
+		}
 	}
 
 	return
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
 	canonicalOrder := canonicalOrderBinary
 	if isSource {
 		canonicalOrder = canonicalOrderSource
@@ -154,6 +168,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	if isRelease {
 		canonicalOrder = canonicalOrderRelease
 	}
+	if isInstaller {
+		canonicalOrder = canonicalOrderInstaller
+	}
 
 	for _, field := range canonicalOrder {
 		value, ok := s[field]
@@ -166,10 +183,21 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		}
 	}
 
-	for field, value := range s {
-		err := writeField(w, field, value, isRelease)
-		if err != nil {
-			return err
+	// no extra fields in installer
+	if !isInstaller {
+		// Print extra fields in deterministic order (alphabetical)
+		keys := make([]string, len(s))
+		i := 0
+		for field := range s {
+			keys[i] = field
+			i++
+		}
+		sort.Strings(keys)
+		for _, field := range keys {
+			err := writeField(w, field, s[field], isRelease)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -212,22 +240,28 @@ func canonicalCase(field string) string {
 
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
-	scanner *bufio.Scanner
+	scanner     *bufio.Scanner
+	isRelease   bool
+	isInstaller bool
 }
 
 // NewControlFileReader creates ControlFileReader, it wraps with buffering
-func NewControlFileReader(r io.Reader) *ControlFileReader {
+func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
 	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
 	scnr.Buffer(nil, MaxFieldSize)
 
-	return &ControlFileReader{scanner: scnr}
+	return &ControlFileReader{
+		scanner:     scnr,
+		isRelease:   isRelease,
+		isInstaller: isInstaller,
+	}
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
+func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
-	lastFieldMultiline := false
+	lastFieldMultiline := c.isInstaller
 
 	for c.scanner.Scan() {
 		line := c.scanner.Text()
@@ -240,7 +274,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 			continue
 		}
 
-		if line[0] == ' ' || line[0] == '\t' {
+		if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
@@ -252,7 +286,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			lastFieldMultiline = isMultilineField(lastField, isRelease)
+			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -15,6 +15,10 @@ type ControlFileSuite struct {
 
 var _ = Suite(&ControlFileSuite{})
 
+const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list
+9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+`
+
 const controlFile = `Package: bti
 Binary: bti
 Version: 032-1
@@ -83,15 +87,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
 
-	stanza1, err := r.ReadStanza(false)
+	stanza1, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza(false)
+	stanza3, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
@@ -103,27 +107,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
-	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza(false)
+	r := NewControlFileReader(s.reader, false, false)
+	stanza, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, true, false)
+	err = stanza.Copy().WriteTo(w, true, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
 
 	str := buf.String()
 
-	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza(false)
+	r = NewControlFileReader(buf, false, false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
+	s.reader = bytes.NewBufferString(installerFile)
+	r := NewControlFileReader(s.reader, false, true)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, false, false, true)
+	c.Assert(err, IsNil)
+	err = w.Flush()
+	c.Assert(err, IsNil)
+
+	r = NewControlFileReader(buf, false, true)
+	stanza2, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Assert(stanza2, DeepEquals, stanza)
+}
+
 func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("Package"), Equals, "Package")
 	c.Check(canonicalCase("package"), Equals, "Package")
@@ -141,8 +165,8 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 	c.Assert(err, IsNil)
 	defer f.Close()
 
-	r := NewControlFileReader(f)
-	stanza, e := r.ReadStanza(false)
+	r := NewControlFileReader(f, false, false)
+	stanza, e := r.ReadStanza()
 	c.Assert(e, IsNil)
 	c.Assert(len(stanza["Provides"]), Equals, 586929)
 }
@@ -150,9 +174,9 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
-		r := NewControlFileReader(reader)
+		r := NewControlFileReader(reader, false, false)
 		for {
-			s, e := r.ReadStanza(false)
+			s, e := r.ReadStanza()
 			if s == nil && e == nil {
 				break
 			}

deb/index_files.go

@@ -29,7 +29,8 @@ type indexFile struct {
 	discardable   bool
 	compressable  bool
 	onlyGzip      bool
-	signable      bool
+	clearSign     bool
+	detachedSign  bool
 	acquireByHash bool
 	relativePath  string
 	tempFilename  string
@@ -77,14 +78,17 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 	file.tempFile.Close()
 
 	exts := []string{""}
+	cksumExts := exts
 	if file.compressable {
 		exts = append(exts, ".gz", ".bz2")
+		cksumExts = exts
 		if file.onlyGzip {
 			exts = []string{".gz"}
+			cksumExts = []string{"", ".gz"}
 		}
 	}
 
-	for _, ext := range exts {
+	for _, ext := range cksumExts {
 		var checksumInfo utils.ChecksumInfo
 
 		checksumInfo, err = utils.ChecksumsForFile(file.tempFilename + ext)
@@ -133,34 +137,42 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		}
 	}
 
-	if file.signable && signer != nil {
-		err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to detached sign file: %s", err)
+	if signer != nil {
+		if file.detachedSign {
+			err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
+			if err != nil {
+				return fmt.Errorf("unable to detached sign file: %s", err)
+			}
+
+			if file.parent.suffix != "" {
+				file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
+					filepath.Join(file.parent.basePath, file.relativePath+".gpg")
+			}
+
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
+				file.tempFilename+".gpg")
+			if err != nil {
+				return fmt.Errorf("unable to publish file: %s", err)
+			}
+
 		}
 
-		err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
-			return fmt.Errorf("unable to clearsign file: %s", err)
-		}
+		if file.clearSign {
+			err = signer.ClearSign(file.tempFilename, filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+			if err != nil {
+				return fmt.Errorf("unable to clearsign file: %s", err)
+			}
 
-		if file.parent.suffix != "" {
-			file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
-				filepath.Join(file.parent.basePath, file.relativePath+".gpg")
-			file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
-				filepath.Join(file.parent.basePath, "In"+file.relativePath)
-		}
+			if file.parent.suffix != "" {
+				file.parent.renameMap[filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix)] =
+					filepath.Join(file.parent.basePath, "In"+file.relativePath)
+			}
 
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
-			file.tempFilename+".gpg")
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
-		}
-
-		err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
-			filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
-		if err != nil {
-			return fmt.Errorf("unable to publish file: %s", err)
+			err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, "In"+file.relativePath+file.parent.suffix),
+				filepath.Join(filepath.Dir(file.tempFilename), "In"+filepath.Base(file.tempFilename)))
+			if err != nil {
+				return fmt.Errorf("unable to publish file: %s", err)
+			}
 		}
 	}
 
@@ -230,11 +242,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 	}
 }
 
-func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false
 	}
-	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
+	key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -244,6 +256,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		} else {
 			if udeb {
 				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else if installer {
+				relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
 			} else {
 				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
 			}
@@ -252,8 +266,9 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		file = &indexFile{
 			parent:        files,
 			discardable:   false,
-			compressable:  true,
-			signable:      false,
+			compressable:  !installer,
+			detachedSign:  installer,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -287,7 +302,8 @@ func (files *indexFiles) ReleaseIndex(component, arch string, udeb bool) *indexF
 			parent:        files,
 			discardable:   udeb,
 			compressable:  false,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -318,7 +334,8 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
 			discardable:   true,
 			compressable:  true,
 			onlyGzip:      true,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -349,7 +366,8 @@ func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile
 			discardable:   true,
 			compressable:  true,
 			onlyGzip:      true,
-			signable:      false,
+			detachedSign:  false,
+			clearSign:     false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,
 		}
@@ -365,19 +383,20 @@ func (files *indexFiles) ReleaseFile() *indexFile {
 		parent:       files,
 		discardable:  false,
 		compressable: false,
-		signable:     true,
+		detachedSign: true,
+		clearSign:    true,
 		relativePath: "Release",
 	}
 }
 
-func (files *indexFiles) FinalizeAll(progress aptly.Progress) (err error) {
+func (files *indexFiles) FinalizeAll(progress aptly.Progress, signer pgp.Signer) (err error) {
 	if progress != nil {
 		progress.InitBar(int64(len(files.indexes)), false)
 		defer progress.ShutdownBar()
 	}
 
 	for _, file := range files.indexes {
-		err = file.Finalize(nil)
+		err = file.Finalize(signer)
 		if err != nil {
 			return
 		}

deb/local.go

@@ -2,12 +2,13 @@ package deb
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"log"
 	"sync"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -93,46 +94,68 @@ func (repo *LocalRepo) RefKey() []byte {
 // LocalRepoCollection does listing, updating/adding/deleting of LocalRepos
 type LocalRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
-	list []*LocalRepo
+	db    database.Storage
+	cache map[string]*LocalRepo
 }
 
 // NewLocalRepoCollection loads LocalRepos from DB and makes up collection
 func NewLocalRepoCollection(db database.Storage) *LocalRepoCollection {
-	result := &LocalRepoCollection{
+	return &LocalRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*LocalRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("L"))
-	result.list = make([]*LocalRepo, 0, len(blobs))
-
-	for _, blob := range blobs {
-		r := &LocalRepo{}
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding repo: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
+func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, unique bool) []*LocalRepo {
+	result := []*LocalRepo(nil)
+	for _, r := range collection.cache {
+		if filter(r) {
+			result = append(result, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+		r := &LocalRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding local repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
-	for _, r := range collection.list {
-		if r.Name == repo.Name {
-			return fmt.Errorf("local repo with name %s already exists", repo.Name)
-		}
+	_, err := collection.ByName(repo.Name)
+
+	if err == nil {
+		return fmt.Errorf("local repo with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -167,58 +190,66 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 
 // ByName looks up repository by name
 func (collection *LocalRepoCollection) ByName(name string) (*LocalRepo, error) {
-	for _, r := range collection.list {
-		if r.Name == name {
-			return r, nil
-		}
+	result := collection.search(func(r *LocalRepo) bool { return r.Name == name }, true)
+	if len(result) == 0 {
+		return nil, fmt.Errorf("local repo with name %s not found", name)
 	}
-	return nil, fmt.Errorf("local repo with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
-	for _, r := range collection.list {
-		if r.UUID == uuid {
-			return r, nil
-		}
+	if r, ok := collection.cache[uuid]; ok {
+		return r, nil
 	}
-	return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+
+	key := (&LocalRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("local repo with uuid %s not found", uuid)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	r := &LocalRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
-	var err error
-	for _, r := range collection.list {
-		err = handler(r)
-		if err != nil {
-			return err
+	return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
+		r := &LocalRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding repo: %s\n", err)
+			return nil
 		}
-	}
-	return err
+
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *LocalRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("L")))
 }
 
 // Drop removes remote repo from collection
 func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
-	repoPosition := -1
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
 		panic("local repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+	delete(collection.cache, repo.UUID)
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/local_test.go

@@ -124,6 +124,11 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewLocalRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 

deb/package.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	gocontext "context"
 	"encoding/json"
 	"fmt"
 	"path/filepath"
@@ -26,6 +27,8 @@ type Package struct {
 	Provides []string
 	// Hash of files section
 	FilesHash uint64
+	// Is this package a dummy installer package
+	IsInstaller bool
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
@@ -43,9 +46,10 @@ type Package struct {
 
 // Package types
 const (
-	PackageTypeBinary = "deb"
-	PackageTypeUdeb   = "udeb"
-	PackageTypeSource = "source"
+	PackageTypeBinary    = "deb"
+	PackageTypeUdeb      = "udeb"
+	PackageTypeSource    = "source"
+	PackageTypeInstaller = "installer"
 )
 
 // Special arhictectures
@@ -168,6 +172,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package {
 	return p
 }
 
+// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file
+func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) {
+	p := &Package{
+		Name:         "installer",
+		Architecture: architecture,
+		IsInstaller:  true,
+		V06Plus:      true,
+		extra:        &Stanza{},
+		deps:         &PackageDependencies{},
+	}
+
+	files := make(PackageFiles, 0)
+	files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	for i := range files {
+		files[i].downloadPath = relPath
+
+		url := repo.PackageURL(files[i].DownloadURL()).String()
+		var size int64
+		size, err = d.GetLength(gocontext.TODO(), url)
+		if err != nil {
+			return nil, err
+		}
+
+		files[i].Checksums.Size = size
+	}
+
+	p.UpdateFiles(files)
+	return p, nil
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -509,6 +548,12 @@ func (p *Package) Stanza() (result Stanza) {
 		if len(sha512) > 0 {
 			result["Checksums-Sha512"] = strings.Join(sha512, "")
 		}
+	} else if p.IsInstaller {
+		sha256 := []string{}
+		for _, f := range p.Files() {
+			sha256 = append(sha256, fmt.Sprintf("%s  %s", f.Checksums.SHA256, f.Filename))
+		}
+		result[""] = strings.Join(sha256, "\n")
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -563,11 +608,7 @@ func (p *Package) Equals(p2 *Package) bool {
 
 // LinkFromPool links package file from pool to dist's pool location
 func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
-	prefix, component string, force bool) error {
-	poolDir, err := p.PoolDirectory()
-	if err != nil {
-		return err
-	}
+	prefix, relPath string, force bool) error {
 
 	for i, f := range p.Files() {
 		sourcePoolPath, err := f.GetPoolPath(packagePool)
@@ -575,7 +616,6 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 			return err
 		}
 
-		relPath := filepath.Join("pool", component, poolDir)
 		publishedDirectory := filepath.Join(prefix, relPath)
 
 		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)

deb/package_files.go

@@ -88,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
 
-func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+// ParseSumField populates PackageFiles by parsing given input
+func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) {
 	for _, line := range strings.Split(input, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
@@ -96,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 		}
 		parts := strings.Fields(line)
 
-		if len(parts) < 3 {
+		if withSize && len(parts) < 3 || !withSize && len(parts) < 2 {
 			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
 		}
 
-		size, err := strconv.ParseInt(parts[1], 10, 64)
-		if err != nil {
-			return nil, fmt.Errorf("unable to parse size: %s", err)
+		var size int64
+		var err error
+		if withSize {
+			size, err = strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse size: %s", err)
+			}
 		}
 
-		filename := filepath.Base(parts[len(parts)-1])
+		filename := parts[len(parts)-1]
+		if onlyBasePath {
+			filename = filepath.Base(filename)
+		}
 
 		found := false
 		pos := 0
@@ -133,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
 	var err error
 
-	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}

deb/package_test.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/http"
 
 	. "gopkg.in/check.v1"
 )
@@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
+	s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza()
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
 }
 
+func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
+	downloader.ExpectResponse("", "udeb.list")
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza()
+	p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader)
+	c.Check(err, IsNil)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
+	c.Check(p.IsInstaller, Equals, true)
+	c.Check(p.Name, Equals, "installer")
+	c.Check(p.Version, Equals, "")
+	c.Check(p.Architecture, Equals, "i386")
+	c.Check(p.Files(), HasLen, 2)
+	c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs")
+	c.Check(p.Files()[0].Checksums.Size, Equals, int64(14))
+	c.Check(p.Files()[1].Filename, Equals, "./udeb.list")
+	c.Check(p.Files()[1].Checksums.Size, Equals, int64(9))
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
@@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "pool/non-free/a/alien-arena", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -490,3 +514,6 @@ Size: 29188
 MD5sum: ae70341c4d96dcded89fa670bcfea31e
 SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
 SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`
+
+const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list`

deb/publish.go

@@ -14,7 +14,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -569,7 +569,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
-			indexes.PackageIndex(component, arch, false)
+			indexes.PackageIndex(component, arch, false, false)
 		}
 
 		if progress != nil {
@@ -585,19 +585,26 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				progress.AddBar(1)
 			}
 
-			matches := false
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					matches = true
-					break
-				}
-			}
+					hadUdebs = hadUdebs || pkg.IsUdeb
 
-			if matches {
-				hadUdebs = hadUdebs || pkg.IsUdeb
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
-				if err != nil {
-					return err
+					var relPath string
+					if !pkg.IsInstaller {
+						poolDir, err2 := pkg.PoolDirectory()
+						if err2 != nil {
+							return err2
+						}
+						relPath = filepath.Join("pool", component, poolDir)
+					} else {
+						relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
+					}
+
+					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite)
+					if err != nil {
+						return err
+					}
+					break
 				}
 			}
 
@@ -612,7 +619,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
-					if !p.SkipContents {
+					if !p.SkipContents && !pkg.IsInstaller {
 						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
 						qualifiedName := []byte(pkg.QualifiedName())
 						contents := pkg.Contents(packagePool, progress)
@@ -629,12 +636,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						}
 					}
 
-					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller)
 					if err != nil {
 						return err
 					}
@@ -687,7 +694,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			// For all architectures, pregenerate .udeb indexes
 			for _, arch := range p.Architectures {
-				indexes.PackageIndex(component, arch, true)
+				indexes.PackageIndex(component, arch, true, false)
 			}
 		}
 
@@ -710,7 +717,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
 				}
 
-				err = release.WriteTo(bufWriter, false, true)
+				err = release.WriteTo(bufWriter, false, true, false)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -742,7 +749,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		progress.Printf("Finalizing metadata files...\n")
 	}
 
-	err = indexes.FinalizeAll(progress)
+	err = indexes.FinalizeAll(progress, signer)
 	if err != nil {
 		return err
 	}
@@ -791,7 +798,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter, false, true)
+	err = release.WriteTo(bufWriter, false, true, false)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}
@@ -852,28 +859,34 @@ type PublishedRepoCollection struct {
 
 // NewPublishedRepoCollection loads PublishedRepos from DB and makes up collection
 func NewPublishedRepoCollection(db database.Storage) *PublishedRepoCollection {
-	result := &PublishedRepoCollection{
+	return &PublishedRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("U"))
-	result.list = make([]*PublishedRepo, 0, len(blobs))
+func (collection *PublishedRepoCollection) loadList() {
+	if collection.list != nil {
+		return
+	}
+
+	blobs := collection.db.FetchByPrefix([]byte("U"))
+	collection.list = make([]*PublishedRepo, 0, len(blobs))
 
 	for _, blob := range blobs {
 		r := &PublishedRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding published repo: %s\n", err)
 		} else {
-			result.list = append(result.list, r)
+			collection.list = append(collection.list, r)
 		}
 	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
+	collection.loadList()
+
 	if collection.CheckDuplicate(repo) != nil {
 		return fmt.Errorf("published repo with storage/prefix/distribution %s/%s/%s already exists", repo.Storage, repo.Prefix, repo.Distribution)
 	}
@@ -889,6 +902,8 @@ func (collection *PublishedRepoCollection) Add(repo *PublishedRepo) error {
 
 // CheckDuplicate verifies that there's no published repo with the same name
 func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *PublishedRepo {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == repo.Prefix && r.Distribution == repo.Distribution && r.Storage == repo.Storage {
 			return r
@@ -978,6 +993,8 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
 
 // ByStoragePrefixDistribution looks up repository by storage, prefix & distribution
 func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage, prefix, distribution string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.Prefix == prefix && r.Distribution == distribution && r.Storage == storage {
 			return r, nil
@@ -991,6 +1008,8 @@ func (collection *PublishedRepoCollection) ByStoragePrefixDistribution(storage,
 
 // ByUUID looks up repository by uuid
 func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo, error) {
+	collection.loadList()
+
 	for _, r := range collection.list {
 		if r.UUID == uuid {
 			return r, nil
@@ -1001,6 +1020,8 @@ func (collection *PublishedRepoCollection) ByUUID(uuid string) (*PublishedRepo,
 
 // BySnapshot looks up repository by snapshot source
 func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceSnapshot {
@@ -1021,6 +1042,8 @@ func (collection *PublishedRepoCollection) BySnapshot(snapshot *Snapshot) []*Pub
 
 // ByLocalRepo looks up repository by local repo source
 func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*PublishedRepo {
+	collection.loadList()
+
 	var result []*PublishedRepo
 	for _, r := range collection.list {
 		if r.SourceKind == SourceLocalRepo {
@@ -1041,18 +1064,21 @@ func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*Publi
 
 // ForEach runs method for each repository
 func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
-	var err error
-	for _, r := range collection.list {
-		err = handler(r)
-		if err != nil {
-			return err
+	return collection.db.ProcessByPrefix([]byte("U"), func(key, blob []byte) error {
+		r := &PublishedRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding published repo: %s\n", err)
+			return nil
 		}
-	}
-	return err
+
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *PublishedRepoCollection) Len() int {
+	collection.loadList()
+
 	return len(collection.list)
 }
 
@@ -1060,6 +1086,8 @@ func (collection *PublishedRepoCollection) Len() int {
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
 	publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
 
+	collection.loadList()
+
 	var err error
 	referencedFiles := map[string][]string{}
 
@@ -1141,6 +1169,9 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
 func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly.PublishedStorageProvider,
 	storage, prefix, distribution string, collectionFactory *CollectionFactory, progress aptly.Progress,
 	force, skipCleanup bool) error {
+
+	collection.loadList()
+
 	repo, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		return err
@@ -1171,7 +1202,10 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 
 	err = repo.RemoveFiles(publishedStorageProvider, removePrefix, removePoolComponents, progress)
 	if err != nil {
-		return err
+		if !force {
+			return fmt.Errorf("published files removal failed, use -force-drop to override: %s", err)
+		}
+		// ignore error with -force-drop
 	}
 
 	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =

deb/publish_test.go

@@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 
-	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza(true)
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(pf)
+	cfr = NewControlFileReader(pf, false, false)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza(false)
+		st, err = cfr.ReadStanza()
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza(false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
 	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(drf)
-	st, err = cfr.ReadStanza(true)
+	cfr = NewControlFileReader(drf, true, false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")

deb/remote.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bytes"
 	gocontext "context"
+	"errors"
 	"fmt"
 	"log"
 	"net/url"
@@ -20,7 +21,7 @@ import (
 	"github.com/aptly-dev/aptly/http"
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -68,6 +69,8 @@ type RemoteRepo struct {
 	DownloadSources bool
 	// Should we download .udebs?
 	DownloadUdebs bool
+	// Should we download installer files?
+	DownloadInstaller bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -78,16 +81,17 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:            uuid.New(),
-		Name:            name,
-		ArchiveRoot:     archiveRoot,
-		Distribution:    distribution,
-		Components:      components,
-		Architectures:   architectures,
-		DownloadSources: downloadSources,
-		DownloadUdebs:   downloadUdebs,
+		UUID:              uuid.New(),
+		Name:              name,
+		ArchiveRoot:       archiveRoot,
+		Distribution:      distribution,
+		Components:        components,
+		Architectures:     architectures,
+		DownloadSources:   downloadSources,
+		DownloadUdebs:     downloadUdebs,
+		DownloadInstaller: downloadInstaller,
 	}
 
 	err := result.prepare()
@@ -140,6 +144,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadUdebs {
 		srcFlag += " [udeb]"
 	}
+	if repo.DownloadInstaller {
+		srcFlag += " [installer]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -243,6 +250,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
 	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
 }
 
+// InstallerPath returns path of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -311,8 +324,8 @@ ok:
 
 	defer release.Close()
 
-	sreader := NewControlFileReader(release)
-	stanza, err := sreader.ReadStanza(true)
+	sreader := NewControlFileReader(release, true, false)
+	stanza, err := sreader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -409,7 +422,7 @@ ok:
 }
 
 // DownloadPackageIndexes downloads & parses package index files
-func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
 	ignoreMismatch bool, maxTries int) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
@@ -420,39 +433,85 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""})
 		if repo.DownloadSources {
-			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture})
 				if repo.DownloadUdebs {
-					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture})
+				}
+				if repo.DownloadInstaller {
+					packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture})
 				}
 			}
 			if repo.DownloadSources {
-				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"})
 			}
 		}
 	}
 
 	for _, info := range packagesPaths {
-		path, kind := info[0], info[1]
+		path, kind, component, architecture := info[0], info[1], info[2], info[3]
 		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+
+		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
-			return err
+			if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
+				// checking if gpg file is only needed when checksums matches are required.
+				// otherwise there actually has been no candidate found and we can continue
+				if ignoreMismatch {
+					continue
+				}
+
+				// some repos do not have installer hashsum file listed in release file but provide a separate gpg file
+				hashsumPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path}).String()
+				packagesFile, err = http.DownloadTemp(gocontext.TODO(), d, hashsumPath)
+				if err != nil {
+					if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+						// installer files are not available in all components and architectures
+						// so ignore it if not found
+						continue
+					}
+
+					return err
+				}
+
+				if verifier != nil {
+					hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
+					var filesig *os.File
+					filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
+					if err != nil {
+						return err
+					}
+
+					err = verifier.VerifyDetachedSignature(filesig, packagesFile, false)
+					if err != nil {
+						return err
+					}
+
+					_, err = packagesFile.Seek(0, 0)
+				}
+
+				packagesReader = packagesFile
+			}
+
+			if err != nil {
+				return err
+			}
 		}
 		defer packagesFile.Close()
 
 		stat, _ := packagesFile.Stat()
 		progress.InitBar(stat.Size(), true)
 
-		sreader := NewControlFileReader(packagesReader)
+		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
 		for {
-			stanza, err := sreader.ReadStanza(false)
+			stanza, err := sreader.ReadStanza()
 			if err != nil {
 				return err
 			}
@@ -474,15 +533,18 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				if err != nil {
 					return err
 				}
-			}
-			err = repo.packageList.Add(p)
-			if err != nil {
-				if _, ok := err.(*PackageConflictError); ok {
-					progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
-				} else {
+			} else if kind == PackageTypeInstaller {
+				p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d)
+				if err != nil {
 					return err
 				}
 			}
+			err = repo.packageList.Add(p)
+			if _, ok := err.(*PackageConflictError); ok {
+				progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
+			} else if err != nil {
+				return err
+			}
 		}
 
 		progress.ShutdownBar()
@@ -654,46 +716,68 @@ func (repo *RemoteRepo) RefKey() []byte {
 // RemoteRepoCollection does listing, updating/adding/deleting of RemoteRepos
 type RemoteRepoCollection struct {
 	*sync.RWMutex
-	db   database.Storage
-	list []*RemoteRepo
+	db    database.Storage
+	cache map[string]*RemoteRepo
 }
 
 // NewRemoteRepoCollection loads RemoteRepos from DB and makes up collection
 func NewRemoteRepoCollection(db database.Storage) *RemoteRepoCollection {
-	result := &RemoteRepoCollection{
+	return &RemoteRepoCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   make(map[string]*RemoteRepo),
 	}
+}
 
-	blobs := db.FetchByPrefix([]byte("R"))
-	result.list = make([]*RemoteRepo, 0, len(blobs))
-
-	for _, blob := range blobs {
-		r := &RemoteRepo{}
-		if err := r.Decode(blob); err != nil {
-			log.Printf("Error decoding mirror: %s\n", err)
-		} else {
-			result.list = append(result.list, r)
+func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, unique bool) []*RemoteRepo {
+	result := []*RemoteRepo(nil)
+	for _, r := range collection.cache {
+		if filter(r) {
+			result = append(result, r)
 		}
 	}
 
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+		r := &RemoteRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding remote repo: %s\n", err)
+			return nil
+		}
+
+		if filter(r) {
+			if _, exists := collection.cache[r.UUID]; !exists {
+				collection.cache[r.UUID] = r
+				result = append(result, r)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
 	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *RemoteRepoCollection) Add(repo *RemoteRepo) error {
-	for _, r := range collection.list {
-		if r.Name == repo.Name {
-			return fmt.Errorf("mirror with name %s already exists", repo.Name)
-		}
+	_, err := collection.ByName(repo.Name)
+
+	if err == nil {
+		return fmt.Errorf("mirror with name %s already exists", repo.Name)
 	}
 
-	err := collection.Update(repo)
+	err = collection.Update(repo)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, repo)
+	collection.cache[repo.UUID] = repo
 	return nil
 }
 
@@ -728,58 +812,65 @@ func (collection *RemoteRepoCollection) LoadComplete(repo *RemoteRepo) error {
 
 // ByName looks up repository by name
 func (collection *RemoteRepoCollection) ByName(name string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
-		if r.Name == name {
-			return r, nil
-		}
+	result := collection.search(func(r *RemoteRepo) bool { return r.Name == name }, true)
+	if len(result) == 0 {
+		return nil, fmt.Errorf("mirror with name %s not found", name)
 	}
-	return nil, fmt.Errorf("mirror with name %s not found", name)
+
+	return result[0], nil
 }
 
 // ByUUID looks up repository by uuid
 func (collection *RemoteRepoCollection) ByUUID(uuid string) (*RemoteRepo, error) {
-	for _, r := range collection.list {
-		if r.UUID == uuid {
-			return r, nil
-		}
+	if r, ok := collection.cache[uuid]; ok {
+		return r, nil
 	}
-	return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+
+	key := (&RemoteRepo{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("mirror with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	r := &RemoteRepo{}
+	err = r.Decode(value)
+
+	if err == nil {
+		collection.cache[r.UUID] = r
+	}
+
+	return r, err
 }
 
 // ForEach runs method for each repository
 func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error) error {
-	var err error
-	for _, r := range collection.list {
-		err = handler(r)
-		if err != nil {
-			return err
+	return collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
+		r := &RemoteRepo{}
+		if err := r.Decode(blob); err != nil {
+			log.Printf("Error decoding mirror: %s\n", err)
+			return nil
 		}
-	}
-	return err
+
+		return handler(r)
+	})
 }
 
 // Len returns number of remote repos
 func (collection *RemoteRepoCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("R")))
 }
 
 // Drop removes remote repo from collection
 func (collection *RemoteRepoCollection) Drop(repo *RemoteRepo) error {
-	repoPosition := -1
-
-	for i, r := range collection.list {
-		if r == repo {
-			repoPosition = i
-			break
-		}
-	}
-
-	if repoPosition == -1 {
+	if _, err := collection.db.Get(repo.Key()); err == database.ErrNotFound {
 		panic("repo not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[repoPosition], collection.list =
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+	delete(collection.cache, repo.UUID)
 
 	err := collection.db.Delete(repo.Key())
 	if err != nil {

deb/remote_test.go

@@ -88,8 +88,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.NewOpenDB(c.MkDir())
@@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 
 	s.repo.DownloadSources = true
 	s.repo.DownloadUdebs = true
+	s.repo.DownloadInstaller = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
 	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
+func (s *RemoteRepoSuite) TestInstallerPath(c *C) {
+	c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS")
+}
+
 func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
 	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
 }
@@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -333,6 +338,49 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	c.Assert(s.repo.packageRefs, NotNil)
 }
 
+func (s *RemoteRepoSuite) TestDownloadWithInstaller(c *C) {
+	s.repo.Architectures = []string{"i386"}
+	s.repo.DownloadInstaller = true
+
+	err := s.repo.Fetch(s.downloader, nil)
+	c.Assert(err, IsNil)
+
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.bz2", &http.Error{Code: 404})
+	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/SHA256SUMS", exampleInstallerHashSumFile)
+	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST", exampleInstallerManifestFile)
+
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
+	c.Assert(err, IsNil)
+	c.Assert(s.downloader.Empty(), Equals, true)
+
+	queue, size, err := s.repo.BuildDownloadQueue(s.packagePool, s.collectionFactory.PackageCollection(), s.cs, false)
+	c.Assert(err, IsNil)
+	c.Check(size, Equals, int64(3)+int64(len(exampleInstallerManifestFile)))
+	c.Check(queue, HasLen, 2)
+
+	q := make([]string, 2)
+	for i := range q {
+		q[i] = queue[i].File.DownloadURL()
+	}
+	sort.Strings(q)
+	c.Check(q[0], Equals, "dists/squeeze/main/installer-i386/current/images/MANIFEST")
+	c.Check(q[1], Equals, "pool/main/a/amanda/amanda-client_3.3.1-3~bpo60+1_amd64.deb")
+
+	s.repo.FinalizeDownload(s.collectionFactory, nil)
+	c.Assert(s.repo.packageRefs, NotNil)
+
+	pkg, err := s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[0])
+	c.Assert(err, IsNil)
+
+	c.Check(pkg.Name, Equals, "amanda-client")
+
+	pkg, err = s.collectionFactory.PackageCollection().ByKey(s.repo.packageRefs.Refs[1])
+	c.Assert(err, IsNil)
+	c.Check(pkg.Name, Equals, "installer")
+}
+
 func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.repo.Architectures = []string{"i386"}
 	s.repo.DownloadSources = true
@@ -347,7 +395,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -391,7 +439,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -416,7 +464,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -440,7 +488,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -468,7 +516,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -490,7 +538,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -521,7 +569,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -567,7 +615,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -593,7 +641,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -628,7 +676,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -646,16 +694,21 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
 	c.Assert(err, IsNil)
+	c.Assert(r, Equals, repo)
+
+	collection := NewRemoteRepoCollection(s.db)
+	r, err = collection.ByUUID(repo.UUID)
+	c.Assert(err, IsNil)
 	c.Assert(r.String(), Equals, repo.String())
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -676,7 +729,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -698,10 +751,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
@@ -874,4 +927,31 @@ SHA1: 66b27417d37e024c46526c2f6d358a754fc552f3
 SHA256: 3608bca1e44ea6c4d268eb6db02260269892c0b42b86bbf1e77a6fa16c3c9282
 `
 
+const exampleInstallerHashSumFile = `82f69d557f0004d2923fb03e4fb47d18187e37768dbfd0c99756f8a6c68a6d3a  ./MANIFEST
+`
+
+const exampleInstallerManifestFile = `cdrom/debian-cd_info.tar.gz     -- isolinux config files for CD
+cdrom/gtk/debian-cd_info.tar.gz -- isolinux help screens for CD (graphical)
+cdrom/gtk/initrd.gz             -- initrd for use with isolinux to build a CD (graphical)
+cdrom/gtk/vmlinuz               -- kernel for use with isolinux to build a CD (graphical)
+cdrom/initrd.gz                 -- initrd for use with isolinux to build a CD
+cdrom/vmlinuz                   -- kernel for use with isolinux to build a CD
+cdrom/xen/debian.cfg            -- example Xen configuration
+cdrom/xen/initrd.gz             -- initrd for installing under Xen
+cdrom/xen/vmlinuz               -- kernel image for installing under Xen
+hd-media/boot.img.gz            -- 1 gb image (compressed) for USB memory stick
+hd-media/gtk/initrd.gz          -- for use on USB memory sticks
+hd-media/gtk/vmlinuz            -- for use on USB memory sticks
+hd-media/initrd.gz              -- for use on USB memory sticks
+hd-media/vmlinuz                -- for use on USB memory sticks
+netboot/debian-installer        -- PXE boot directory for tftp server
+netboot/gtk/debian-installer    -- PXE boot directory for tftp server (graphical installer)
+netboot/gtk/mini.iso            -- not so tiny CD image that boots the graphical netboot installer
+netboot/gtk/netboot.tar.gz      -- tarball of PXE boot directory (graphical installer)
+netboot/mini.iso                -- tiny CD image that boots the netboot installer
+netboot/netboot.tar.gz          -- tarball of PXE boot directory
+netboot/xen/debian.cfg          -- example Xen configuration
+netboot/xen/initrd.gz           -- initrd for installing under Xen
+netboot/xen/vmlinuz             -- kernel image for installing under Xen`
+
 const exampleSourcesFile = sourcePackageMeta

deb/snapshot.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/go-uuid/uuid"
+	"github.com/pborman/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -173,50 +173,36 @@ func (s *Snapshot) Decode(input []byte) error {
 // SnapshotCollection does listing, updating/adding/deleting of Snapshots
 type SnapshotCollection struct {
 	*sync.RWMutex
-	db   database.Storage
-	list []*Snapshot
+	db    database.Storage
+	cache map[string]*Snapshot
 }
 
 // NewSnapshotCollection loads Snapshots from DB and makes up collection
 func NewSnapshotCollection(db database.Storage) *SnapshotCollection {
-	result := &SnapshotCollection{
+	return &SnapshotCollection{
 		RWMutex: &sync.RWMutex{},
 		db:      db,
+		cache:   map[string]*Snapshot{},
 	}
-
-	blobs := db.FetchByPrefix([]byte("S"))
-	result.list = make([]*Snapshot, 0, len(blobs))
-
-	for _, blob := range blobs {
-		s := &Snapshot{}
-		if err := s.Decode(blob); err != nil {
-			log.Printf("Error decoding snapshot: %s\n", err)
-		} else {
-			result.list = append(result.list, s)
-		}
-	}
-
-	return result
 }
 
 // Add appends new repo to collection and saves it
 func (collection *SnapshotCollection) Add(snapshot *Snapshot) error {
-	for _, s := range collection.list {
-		if s.Name == snapshot.Name {
-			return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
-		}
+	_, err := collection.ByName(snapshot.Name)
+	if err == nil {
+		return fmt.Errorf("snapshot with name %s already exists", snapshot.Name)
 	}
 
-	err := collection.Update(snapshot)
+	err = collection.Update(snapshot)
 	if err != nil {
 		return err
 	}
 
-	collection.list = append(collection.list, snapshot)
+	collection.cache[snapshot.UUID] = snapshot
 	return nil
 }
 
-// Update stores updated information about repo in DB
+// Update stores updated information about snapshot in DB
 func (collection *SnapshotCollection) Update(snapshot *Snapshot) error {
 	err := collection.db.Put(snapshot.Key(), snapshot.Encode())
 	if err != nil {
@@ -239,83 +225,132 @@ func (collection *SnapshotCollection) LoadComplete(snapshot *Snapshot) error {
 	return snapshot.packageRefs.Decode(encoded)
 }
 
-// ByName looks up snapshot by name
-func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
-	for _, s := range collection.list {
-		if s.Name == name {
-			return s, nil
+func (collection *SnapshotCollection) search(filter func(*Snapshot) bool, unique bool) []*Snapshot {
+	result := []*Snapshot(nil)
+	for _, s := range collection.cache {
+		if filter(s) {
+			result = append(result, s)
 		}
 	}
+
+	if unique && len(result) > 0 {
+		return result
+	}
+
+	collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+			return nil
+		}
+
+		if filter(s) {
+			if _, exists := collection.cache[s.UUID]; !exists {
+				collection.cache[s.UUID] = s
+				result = append(result, s)
+				if unique {
+					return errors.New("abort")
+				}
+			}
+		}
+
+		return nil
+	})
+
+	return result
+}
+
+// ByName looks up snapshot by name
+func (collection *SnapshotCollection) ByName(name string) (*Snapshot, error) {
+	result := collection.search(func(s *Snapshot) bool { return s.Name == name }, true)
+	if len(result) > 0 {
+		return result[0], nil
+	}
+
 	return nil, fmt.Errorf("snapshot with name %s not found", name)
 }
 
 // ByUUID looks up snapshot by UUID
 func (collection *SnapshotCollection) ByUUID(uuid string) (*Snapshot, error) {
-	for _, s := range collection.list {
-		if s.UUID == uuid {
-			return s, nil
-		}
+	if s, ok := collection.cache[uuid]; ok {
+		return s, nil
 	}
-	return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+
+	key := (&Snapshot{UUID: uuid}).Key()
+
+	value, err := collection.db.Get(key)
+	if err == database.ErrNotFound {
+		return nil, fmt.Errorf("snapshot with uuid %s not found", uuid)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	s := &Snapshot{}
+	err = s.Decode(value)
+
+	if err == nil {
+		collection.cache[s.UUID] = s
+	}
+
+	return s, err
 }
 
 // ByRemoteRepoSource looks up snapshots that have specified RemoteRepo as a source
 func (collection *SnapshotCollection) ByRemoteRepoSource(repo *RemoteRepo) []*Snapshot {
-	var result []*Snapshot
-
-	for _, s := range collection.list {
-		if s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
+	return collection.search(func(s *Snapshot) bool {
+		return s.SourceKind == SourceRemoteRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
+	}, false)
 }
 
 // ByLocalRepoSource looks up snapshots that have specified LocalRepo as a source
 func (collection *SnapshotCollection) ByLocalRepoSource(repo *LocalRepo) []*Snapshot {
-	var result []*Snapshot
-
-	for _, s := range collection.list {
-		if s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
+	return collection.search(func(s *Snapshot) bool {
+		return s.SourceKind == SourceLocalRepo && utils.StrSliceHasItem(s.SourceIDs, repo.UUID)
+	}, false)
 }
 
 // BySnapshotSource looks up snapshots that have specified snapshot as a source
 func (collection *SnapshotCollection) BySnapshotSource(snapshot *Snapshot) []*Snapshot {
-	var result []*Snapshot
-
-	for _, s := range collection.list {
-		if s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID) {
-			result = append(result, s)
-		}
-	}
-	return result
+	return collection.search(func(s *Snapshot) bool {
+		return s.SourceKind == "snapshot" && utils.StrSliceHasItem(s.SourceIDs, snapshot.UUID)
+	}, false)
 }
 
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) ForEach(handler func(*Snapshot) error) error {
-	var err error
-	for _, s := range collection.list {
-		err = handler(s)
-		if err != nil {
-			return err
+	return collection.db.ProcessByPrefix([]byte("S"), func(key, blob []byte) error {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+			return nil
 		}
-	}
-	return err
+
+		return handler(s)
+	})
 }
 
 // ForEachSorted runs method for each snapshot following some sort order
 func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler func(*Snapshot) error) error {
-	sorter, err := newSnapshotSorter(sortMethod, collection)
+	blobs := collection.db.FetchByPrefix([]byte("S"))
+	list := make([]*Snapshot, 0, len(blobs))
+
+	for _, blob := range blobs {
+		s := &Snapshot{}
+		if err := s.Decode(blob); err != nil {
+			log.Printf("Error decoding snapshot: %s\n", err)
+		} else {
+			list = append(list, s)
+		}
+	}
+
+	sorter, err := newSnapshotSorter(sortMethod, list)
 	if err != nil {
 		return err
 	}
 
-	for _, i := range sorter.list {
-		err = handler(collection.list[i])
+	for _, s := range sorter.list {
+		err = handler(s)
 		if err != nil {
 			return err
 		}
@@ -327,26 +362,16 @@ func (collection *SnapshotCollection) ForEachSorted(sortMethod string, handler f
 // Len returns number of snapshots in collection
 // ForEach runs method for each snapshot
 func (collection *SnapshotCollection) Len() int {
-	return len(collection.list)
+	return len(collection.db.KeysByPrefix([]byte("S")))
 }
 
 // Drop removes snapshot from collection
 func (collection *SnapshotCollection) Drop(snapshot *Snapshot) error {
-	snapshotPosition := -1
-
-	for i, s := range collection.list {
-		if s == snapshot {
-			snapshotPosition = i
-			break
-		}
-	}
-
-	if snapshotPosition == -1 {
+	if _, err := collection.db.Get(snapshot.Key()); err == database.ErrNotFound {
 		panic("snapshot not found!")
 	}
 
-	collection.list[len(collection.list)-1], collection.list[snapshotPosition], collection.list =
-		nil, collection.list[len(collection.list)-1], collection.list[:len(collection.list)-1]
+	delete(collection.cache, snapshot.UUID)
 
 	err := collection.db.Delete(snapshot.Key())
 	if err != nil {
@@ -363,13 +388,12 @@ const (
 )
 
 type snapshotSorter struct {
-	list       []int
-	collection *SnapshotCollection
+	list       []*Snapshot
 	sortMethod int
 }
 
-func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snapshotSorter, error) {
-	s := &snapshotSorter{collection: collection}
+func newSnapshotSorter(sortMethod string, list []*Snapshot) (*snapshotSorter, error) {
+	s := &snapshotSorter{list: list}
 
 	switch sortMethod {
 	case "time", "Time":
@@ -380,11 +404,6 @@ func newSnapshotSorter(sortMethod string, collection *SnapshotCollection) (*snap
 		return nil, fmt.Errorf("sorting method \"%s\" unknown", sortMethod)
 	}
 
-	s.list = make([]int, len(collection.list))
-	for i := range s.list {
-		s.list[i] = i
-	}
-
 	sort.Sort(s)
 
 	return s, nil
@@ -397,9 +416,9 @@ func (s *snapshotSorter) Swap(i, j int) {
 func (s *snapshotSorter) Less(i, j int) bool {
 	switch s.sortMethod {
 	case SortName:
-		return s.collection.list[s.list[i]].Name < s.collection.list[s.list[j]].Name
+		return s.list[i].Name < s.list[j].Name
 	case SortTime:
-		return s.collection.list[s.list[i]].CreatedAt.Before(s.collection.list[s.list[j]].CreatedAt)
+		return s.list[i].CreatedAt.Before(s.list[j].CreatedAt)
 	}
 	panic("unknown sort method")
 }

deb/snapshot_bench_test.go

@@ -0,0 +1,98 @@
+package deb
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+func BenchmarkSnapshotCollectionForEach(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		collection.ForEach(func(s *Snapshot) error {
+			return nil
+		})
+	}
+}
+
+func BenchmarkSnapshotCollectionByUUID(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	uuids := []string{}
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+		uuids = append(uuids, snapshot.UUID)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		if _, err := collection.ByUUID(uuids[i%len(uuids)]); err != nil {
+			b.FailNow()
+		}
+	}
+}
+
+func BenchmarkSnapshotCollectionByName(b *testing.B) {
+	const count = 1024
+
+	tmpDir := os.TempDir()
+	defer os.RemoveAll(tmpDir)
+
+	db, _ := database.NewOpenDB(tmpDir)
+	defer db.Close()
+
+	collection := NewSnapshotCollection(db)
+
+	for i := 0; i < count; i++ {
+		snapshot := NewSnapshotFromRefList(fmt.Sprintf("snapshot%d", i), nil, NewPackageRefList(), fmt.Sprintf("Snapshot number %d", i))
+		if collection.Add(snapshot) != nil {
+			b.FailNow()
+		}
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		collection = NewSnapshotCollection(db)
+
+		if _, err := collection.ByName(fmt.Sprintf("snapshot%d", i%count)); err != nil {
+			b.FailNow()
+		}
+	}
+}

deb/snapshot_test.go

@@ -2,6 +2,7 @@ package deb
 
 import (
 	"errors"
+	"sort"
 
 	"github.com/aptly-dev/aptly/database"
 
@@ -17,7 +18,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -116,11 +117,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -158,6 +159,10 @@ func (s *SnapshotCollectionSuite) TestAddByNameByUUID(c *C) {
 	snapshot, err = collection.ByUUID(s.snapshot1.UUID)
 	c.Assert(err, IsNil)
 	c.Assert(snapshot.String(), Equals, s.snapshot1.String())
+
+	snapshot, err = collection.ByUUID(s.snapshot2.UUID)
+	c.Assert(err, IsNil)
+	c.Assert(snapshot.String(), Equals, s.snapshot2.String())
 }
 
 func (s *SnapshotCollectionSuite) TestUpdateLoadComplete(c *C) {
@@ -193,6 +198,23 @@ func (s *SnapshotCollectionSuite) TestForEachAndLen(c *C) {
 	c.Assert(err, Equals, e)
 }
 
+func (s *SnapshotCollectionSuite) TestForEachSorted(c *C) {
+	s.collection.Add(s.snapshot2)
+	s.collection.Add(s.snapshot1)
+	s.collection.Add(s.snapshot4)
+	s.collection.Add(s.snapshot3)
+
+	names := []string{}
+
+	err := s.collection.ForEachSorted("name", func(snapshot *Snapshot) error {
+		names = append(names, snapshot.Name)
+		return nil
+	})
+	c.Assert(err, IsNil)
+
+	c.Check(sort.StringsAreSorted(names), Equals, true)
+}
+
 func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Assert(s.collection.Add(s.snapshot1), IsNil)
 	c.Assert(s.collection.Add(s.snapshot2), IsNil)
@@ -200,7 +222,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil))
 }
@@ -230,7 +252,11 @@ func (s *SnapshotCollectionSuite) TestFindSnapshotSource(c *C) {
 	c.Assert(s.collection.Add(snapshot4), IsNil)
 	c.Assert(s.collection.Add(snapshot5), IsNil)
 
-	c.Check(s.collection.BySnapshotSource(s.snapshot1), DeepEquals, []*Snapshot{snapshot3, snapshot4})
+	list := s.collection.BySnapshotSource(s.snapshot1)
+	sorter, _ := newSnapshotSorter("name", list)
+	sort.Sort(sorter)
+
+	c.Check(sorter.list, DeepEquals, []*Snapshot{snapshot3, snapshot4})
 	c.Check(s.collection.BySnapshotSource(s.snapshot2), DeepEquals, []*Snapshot{snapshot3})
 	c.Check(s.collection.BySnapshotSource(snapshot5), DeepEquals, []*Snapshot(nil))
 }

deb/testdata/changes/calamares.changes

@@ -0,0 +1,32 @@
+Format: 1.8
+Date: Thu, 27 Nov 2014 13:24:53 +0000
+Source: calamares
+Binary: calamares calamares-dbg
+Architecture: source amd64
+Version: 0+git20141127.99
+Distribution: sid
+Urgency: medium
+Maintainer: Rohan Garg <rohan@kde.org>
+Changed-By: Rohan <rohan@kde.org>
+Description:
+ calamares  - distribution-independent installer framework
+ calamares-dbg - distribution-independent installer framework -- debug symbols
+Changes:
+ calamares (0+git20141127.99) sid; urgency=medium
+ .
+   * Update from git
+Checksums-Sha1:
+ 79f10e955dab6eb25b7f7bae18213f367a3a0396 1106 calamares_0+git20141127.99.dsc
+ 294c28e2c8e34e72ca9ee0d9da5c14f3bf4188db 2694800 calamares_0+git20141127.99.tar.xz
+ d6c26c04b5407c7511f61cb3e3de60c4a1d6c4ff 1698924 calamares_0+git20141127.99_amd64.deb
+ a3da632d193007b0d4a1aff73159fde1b532d7a8 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Checksums-Sha256:
+ 35b3280a7b1ffe159a276128cb5c408d687318f60ecbb8ab6dedb2e49c4e82dc 1106 calamares_0+git20141127.99.dsc
+ 5576b9caaf814564830f95561227e4f04ee87b31da22c1371aab155cbf7ce395 2694800 calamares_0+git20141127.99.tar.xz
+ 2e6e2f232ed7ffe52369928ebdf5436d90feb37840286ffba79e87d57a43a2e9 1698924 calamares_0+git20141127.99_amd64.deb
+ 8dd926080ed7bad2e2439e37e49ce12d5f1357c5041b7da4d860a1041f878a8a 12835902 calamares-dbg_0+git20141127.99_amd64.deb
+Files:
+ 05fd8f3ffe8f362c5ef9bad2f936a56e 1106 devel optional calamares_0+git20141127.99.dsc
+ 097e55c81abd8e5f30bb2eed90c2c1e9 2694800 devel optional calamares_0+git20141127.99.tar.xz
+ 827fb3b12534241e119815d331e8197b 1698924 devel optional calamares_0+git20141127.99_amd64.deb
+ e6f8ce70f564d1f68cb57758b15b13e3 12835902 debug optional calamares-dbg_0+git20141127.99_amd64.deb

deb/testdata/changes/hardlink_0.2.1-invalidfiles_amd64.changes

@@ -0,0 +1,22 @@
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 invalidhardlink_0.2.1.dsc
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 invalidhardlink_0.2.1.dsc
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional invalidhardlink_0.2.1.dsc

deb/testdata/changes/hardlink_0.2.1-invalidsig_amd64.changes

@@ -0,0 +1,39 @@
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA1
+
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ ff306b8f923653b78e00c45ebbc6c1c734859cdf 949 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ c0d7458aa2ca3886cd6885f395a289efbc9a396e6765cbbca45f51fde859ea70 949 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Files: 
+ 4efce26825af5842f43961096dd890b3 949 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+
+-----BEGIN PGP SIGNATURE-----
+Version: GnuPG v1.4.12 (GNU/Linux)
+
+iEYEARECAAYFAlUFwywACgkQIdu4nBbbPm1DLACgwW4V8qLQC/QHC/7+t3Iq47Ez
+eesAn3ZYLQvLYRw3wPTKVAPI+AW6Fjxi
+=hRBo
+-----END PGP SIGNATURE-----

deb/testdata/changes/hardlink_0.2.1.dsc

@@ -0,0 +1,19 @@
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: any
+Version: 0.2.1
+Maintainer: Julian Andres Klode <jak@debian.org>
+Homepage: http://jak-linux.org/projects/hardlink/
+Standards-Version: 3.9.3
+Vcs-Browser: http://git.debian.org/?p=users/jak/hardlink.git;a=summary
+Vcs-Git: git://git.debian.org/git/users/jak/hardlink.git
+Build-Depends: debhelper (>= 9), pkg-config, libpcre3-dev
+Package-List: 
+ hardlink deb utils optional
+Checksums-Sha1: 
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+Checksums-Sha256: 
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+Files: 
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 hardlink_0.2.1.tar.gz

deb/testdata/changes/hardlink_0.2.1_amd64.buildinfo

@@ -0,0 +1,170 @@
+Format: 1.0
+Source: hardlink
+Binary: hardlink
+Architecture: amd64
+Version: 0.2.0
+Checksums-Md5:
+ 2081e20b36c47f82811c25841cc0e41b 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha1: 
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+Checksums-Sha256: 
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+Build-Origin: Debian
+Build-Architecture: amd64
+Build-Kernel-Version: 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07)
+Build-Date: Sun, 21 Jul 2019 08:01:03 +1400
+Build-Path: /build/hardlink-0.3.0/2nd
+Installed-Build-Depends:
+ autoconf (= 2.69-11),
+ automake (= 1:1.15.1-3.1),
+ autopoint (= 0.19.8.1-6),
+ autotools-dev (= 20180224.1),
+ base-files (= 10.1),
+ base-passwd (= 3.5.45),
+ bash (= 4.4.18-3),
+ binutils (= 2.30-21),
+ binutils-common (= 2.30-21),
+ binutils-x86-64-linux-gnu (= 2.30-21),
+ bsdmainutils (= 11.1.2+b1),
+ bsdutils (= 1:2.32-0.1),
+ build-essential (= 12.5),
+ bzip2 (= 1.0.6-8.1),
+ coreutils (= 8.28-1),
+ cpp (= 4:7.3.0-3),
+ cpp-7 (= 7.3.0-26+really21.0~reproducible0),
+ dash (= 0.5.8-2.10),
+ debconf (= 1.5.67),
+ debhelper (= 11.3.2),
+ debianutils (= 4.8.6),
+ dh-autoreconf (= 19),
+ dh-strip-nondeterminism (= 0.042-1),
+ diffutils (= 1:3.6-1),
+ dpkg (= 1.19.0.5.0~reproducible1),
+ dpkg-dev (= 1.19.0.5.0~reproducible1),
+ dwz (= 0.12-2),
+ fdisk (= 2.32-0.1),
+ file (= 1:5.33-3),
+ findutils (= 4.6.0+git+20171230-2),
+ g++ (= 4:7.3.0-3),
+ g++-7 (= 7.3.0-26+really21.0~reproducible0),
+ gcc (= 4:7.3.0-3),
+ gcc-7 (= 7.3.0-26+really21.0~reproducible0),
+ gcc-7-base (= 7.3.0-26+really21.0~reproducible0),
+ gcc-8-base (= 8.1.0-6),
+ gettext (= 0.19.8.1-6+b1),
+ gettext-base (= 0.19.8.1-6+b1),
+ grep (= 3.1-2),
+ groff-base (= 1.22.3-10),
+ gzip (= 1.6-5+b1),
+ hostname (= 3.20),
+ init-system-helpers (= 1.51),
+ intltool-debian (= 0.35.0+20060710.4),
+ libacl1 (= 2.2.52-3+b1),
+ libarchive-zip-perl (= 1.60-1),
+ libasan4 (= 7.3.0-26+really21.0~reproducible0),
+ libatomic1 (= 8.1.0-6),
+ libattr1 (= 1:2.4.47-2+b2),
+ libattr1-dev (= 1:2.4.47-2+b2),
+ libaudit-common (= 1:2.8.3-1),
+ libaudit1 (= 1:2.8.3-1),
+ libbinutils (= 2.30-21),
+ libblkid1 (= 2.32-0.1),
+ libbsd0 (= 0.9.1-1),
+ libbz2-1.0 (= 1.0.6-8.1),
+ libc-bin (= 2.27-3),
+ libc-dev-bin (= 2.27-3),
+ libc6 (= 2.27-3),
+ libc6-dev (= 2.27-3),
+ libcap-ng0 (= 0.7.9-1),
+ libcc1-0 (= 8.1.0-6),
+ libcilkrts5 (= 7.3.0-26+really21.0~reproducible0),
+ libcroco3 (= 0.6.12-2),
+ libdb5.3 (= 5.3.28-13.1+b1),
+ libdebconfclient0 (= 0.243),
+ libdpkg-perl (= 1.19.0.5.0~reproducible1),
+ libelf1 (= 0.170-0.4),
+ libfdisk1 (= 2.32-0.1),
+ libffi6 (= 3.2.1-8),
+ libfile-stripnondeterminism-perl (= 0.042-1),
+ libfreetype6 (= 2.8.1-2),
+ libgcc-7-dev (= 7.3.0-26+really21.0~reproducible0),
+ libgcc1 (= 1:8.1.0-6),
+ libgcrypt20 (= 1.8.3-1),
+ libgdbm-compat4 (= 1.14.1-6+b1),
+ libgdbm5 (= 1.14.1-6+b1),
+ libglib2.0-0 (= 2.56.1-2),
+ libgmp10 (= 2:6.1.2+dfsg-3),
+ libgomp1 (= 8.1.0-6),
+ libgpg-error0 (= 1.31-1),
+ libgraphite2-3 (= 1.3.11-2),
+ libharfbuzz0b (= 1.7.6-1+b1),
+ libicu-le-hb0 (= 1.0.3+git161113-5),
+ libicu60 (= 60.2-6),
+ libisl19 (= 0.19-1),
+ libitm1 (= 8.1.0-6),
+ liblsan0 (= 8.1.0-6),
+ liblz4-1 (= 1.8.2-1),
+ liblzma5 (= 5.2.2-1.3),
+ libmagic-mgc (= 1:5.33-3),
+ libmagic1 (= 1:5.33-3),
+ libmount1 (= 2.32-0.1),
+ libmpc3 (= 1.1.0-1),
+ libmpfr6 (= 4.0.1-1),
+ libmpx2 (= 8.1.0-6),
+ libncurses6 (= 6.1+20180210-4),
+ libncursesw6 (= 6.1+20180210-4),
+ libpam-modules (= 1.1.8-3.7),
+ libpam-modules-bin (= 1.1.8-3.7),
+ libpam-runtime (= 1.1.8-3.7),
+ libpam0g (= 1.1.8-3.7),
+ libpcre16-3 (= 2:8.39-9),
+ libpcre3 (= 2:8.39-9),
+ libpcre3-dev (= 2:8.39-9),
+ libpcre32-3 (= 2:8.39-9),
+ libpcrecpp0v5 (= 2:8.39-9),
+ libperl5.26 (= 5.26.2-6),
+ libpipeline1 (= 1.5.0-1),
+ libpng16-16 (= 1.6.34-1),
+ libquadmath0 (= 8.1.0-6),
+ libseccomp2 (= 2.3.3-2),
+ libselinux1 (= 2.8-1),
+ libsigsegv2 (= 2.12-2),
+ libsmartcols1 (= 2.32-0.1),
+ libstdc++-7-dev (= 7.3.0-26+really21.0~reproducible0),
+ libstdc++6 (= 8.1.0-6),
+ libsystemd0 (= 238-5),
+ libtimedate-perl (= 2.3000-2),
+ libtinfo6 (= 6.1+20180210-4),
+ libtool (= 2.4.6-2.1),
+ libtsan0 (= 8.1.0-6),
+ libubsan0 (= 7.3.0-26+really21.0~reproducible0),
+ libudev1 (= 238-5),
+ libunistring2 (= 0.9.8-1),
+ libuuid1 (= 2.32-0.1),
+ libxml2 (= 2.9.4+dfsg1-7),
+ linux-libc-dev (= 4.16.12-1),
+ login (= 1:4.5-1),
+ m4 (= 1.4.18-1),
+ make (= 4.2.1-1),
+ man-db (= 2.8.3-2),
+ mawk (= 1.3.3-17+b3),
+ ncurses-base (= 6.1+20180210-4),
+ ncurses-bin (= 6.1+20180210-4),
+ patch (= 2.7.6-2),
+ perl (= 5.26.2-6),
+ perl-base (= 5.26.2-6),
+ perl-modules-5.26 (= 5.26.2-6),
+ pkg-config (= 0.29-4+b1),
+ po-debconf (= 1.0.20),
+ sed (= 4.4-2),
+ sysvinit-utils (= 2.88dsf-59.10),
+ tar (= 1.30+dfsg-2),
+ util-linux (= 2.32-0.1),
+ xz-utils (= 5.2.2-1.3),
+ zlib1g (= 1:1.2.11.dfsg-1)
+Environment:
+ BUILD_PATH_PREFIX_MAP="hardlink_0.3.0=/build/hardlink-0.3.0/2nd"
+ DEB_BUILD_OPTIONS="buildinfo=+all parallel=16"
+ LANG="C"
+ LC_ALL="C"
+ SOURCE_DATE_EPOCH="1411647982"

deb/testdata/changes/hardlink_0.2.1_amd64.changes

@@ -0,0 +1,34 @@
+Format: 1.8
+Date: Sat, 12 May 2014 12:57:02 +0200
+Source: hardlink
+Binary: hardlink
+Architecture: source amd64
+Version: 0.2.1
+Distribution: unstable
+Urgency: low
+Maintainer: Julian Andres Klode <jak@debian.org>
+Changed-By: Aptly Tester (don't use it) <test@aptly.info>
+Description: 
+ hardlink   - Hardlinks multiple copies of the same file
+Changes: 
+ hardlink (0.2.1) unstable; urgency=low
+ .
+   * Update just to try it out :)
+Checksums-Sha1: 
+ d20d6820bfccf5e3a5b120eedabba96e71da60ff 703 hardlink_0.2.1.dsc
+ 6e95b8cba450343ab4dc01902e521f29fbd87ac2 12516 hardlink_0.2.1.tar.gz
+ 1ac0e962854dff46f14fa7943746660d3cad1679 12468 hardlink_0.2.1_amd64.deb
+ 06c38a55e81907e573641d7f979f428583bd3165 4885 hardlink_0.2.1_amd64.buildinfo
+ be2b66e32b29ab654190e7ecb4cc31f48ec061cb 12084 hardlink_0.2.0_i386.deb
+Checksums-Sha256: 
+ 19d08cfadd58aee05fd06aca261a873bafeb55cac58bf4916cc070f5821803e7 703 hardlink_0.2.1.dsc
+ 4df0adce005526a1f0e1b38171ddb1f017faae9205f5b1c6dfb0fb4207767271 12516 hardlink_0.2.1.tar.gz
+ 668399580590bf1ffcd9eb161b6e574751e15f71820c6e08245dac7c5111a0ee 12468 hardlink_0.2.1_amd64.deb
+ 2a4905626b8664868b712a5b3d21f44c2665d17f9a4b67dc2e2bfd03f4ec54ee 4885 hardlink_0.2.1_amd64.buildinfo
+ a9d657d9413029e484c2a8c059c95a3e31eb1b47eca96455c1b9c330352293c9 12084 hardlink_0.2.0_i386.deb
+Files: 
+ b75fe0616f24deb28a3017c1dbae219a 703 utils optional hardlink_0.2.1.dsc
+ 8e2caa4d82f228bac08dc9a38bc6edb3 12516 utils optional hardlink_0.2.1.tar.gz
+ 2081e20b36c47f82811c25841cc0e41b 12468 utils optional hardlink_0.2.1_amd64.deb
+ 9468a19700d038fa6cd10c5e42063083 12084 utils optional hardlink_0.2.0_i386.deb
+ 490174686848eca6da4e5f79d899efe7 4885 utils optional hardlink_0.2.1_amd64.buildinfo

deb/version_test.go

@@ -5,7 +5,6 @@ import (
 )
 
 type VersionSuite struct {
-	stanza Stanza
 }
 
 var _ = Suite(&VersionSuite{})