Add support to mirror non package installer files

deb/changes.go

@@ -95,8 +95,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		text = input
 	}
 
-	reader := NewControlFileReader(text)
-	c.Stanza, err = reader.ReadStanza(false)
+	reader := NewControlFileReader(text, false, false)
+	c.Stanza, err = reader.ReadStanza()
 	if err != nil {
 		return err
 	}

deb/deb.go

@@ -89,8 +89,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				}
 
 				if tarHeader.Name == "./control" || tarHeader.Name == "control" {
-					reader := NewControlFileReader(untar)
-					stanza, err := reader.ReadStanza(false)
+					reader := NewControlFileReader(untar, false, false)
+					stanza, err := reader.ReadStanza()
 					if err != nil {
 						return nil, err
 					}
@@ -129,8 +129,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		text = file
 	}
 
-	reader := NewControlFileReader(text)
-	stanza, err := reader.ReadStanza(false)
+	reader := NewControlFileReader(text, false, false)
+	stanza, err := reader.ReadStanza()
 	if err != nil {
 		return nil, err
 	}

deb/format.go

@@ -88,6 +88,9 @@ var (
 		"Directory",
 		"Files",
 	}
+	canonicalOrderInstaller = []string{
+		"",
+	}
 )
 
 // Copy returns copy of Stanza
@@ -101,6 +104,9 @@ func (s Stanza) Copy() (result Stanza) {
 
 func isMultilineField(field string, isRelease bool) bool {
 	switch field {
+	// file without a section
+	case "":
+		return true
 	case "Description":
 		return true
 	case "Files":
@@ -132,21 +138,26 @@ func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error
 	if !isMultilineField(field, isRelease) {
 		_, err = w.WriteString(field + ": " + value + "\n")
 	} else {
-		if !strings.HasSuffix(value, "\n") {
+		if field != "" && !strings.HasSuffix(value, "\n") {
 			value = value + "\n"
 		}
 
-		if field != "Description" {
+		if field != "Description" && field != "" {
 			value = "\n" + value
 		}
-		_, err = w.WriteString(field + ":" + value)
+
+		if field != "" {
+			_, err = w.WriteString(field + ":" + value)
+		} else {
+			_, err = w.WriteString(value)
+		}
 	}
 
 	return
 }
 
 // WriteTo saves stanza back to stream, modifying itself on the fly
-func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
+func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error {
 	canonicalOrder := canonicalOrderBinary
 	if isSource {
 		canonicalOrder = canonicalOrderSource
@@ -154,6 +165,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 	if isRelease {
 		canonicalOrder = canonicalOrderRelease
 	}
+	if isInstaller {
+		canonicalOrder = canonicalOrderInstaller
+	}
 
 	for _, field := range canonicalOrder {
 		value, ok := s[field]
@@ -166,10 +180,13 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error {
 		}
 	}
 
-	for field, value := range s {
-		err := writeField(w, field, value, isRelease)
-		if err != nil {
-			return err
+	// no extra fields in installer
+	if !isInstaller {
+		for field, value := range s {
+			err := writeField(w, field, value, isRelease)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -212,22 +229,28 @@ func canonicalCase(field string) string {
 
 // ControlFileReader implements reading of control files stanza by stanza
 type ControlFileReader struct {
-	scanner *bufio.Scanner
+	scanner     *bufio.Scanner
+	isRelease   bool
+	isInstaller bool
 }
 
 // NewControlFileReader creates ControlFileReader, it wraps with buffering
-func NewControlFileReader(r io.Reader) *ControlFileReader {
+func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader {
 	scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768))
 	scnr.Buffer(nil, MaxFieldSize)
 
-	return &ControlFileReader{scanner: scnr}
+	return &ControlFileReader{
+		scanner:     scnr,
+		isRelease:   isRelease,
+		isInstaller: isInstaller,
+	}
 }
 
 // ReadStanza reeads one stanza from control file
-func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
+func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 	stanza := make(Stanza, 32)
 	lastField := ""
-	lastFieldMultiline := false
+	lastFieldMultiline := c.isInstaller
 
 	for c.scanner.Scan() {
 		line := c.scanner.Text()
@@ -240,7 +263,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 			continue
 		}
 
-		if line[0] == ' ' || line[0] == '\t' {
+		if line[0] == ' ' || line[0] == '\t' || c.isInstaller {
 			if lastFieldMultiline {
 				stanza[lastField] += line + "\n"
 			} else {
@@ -252,7 +275,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) {
 				return nil, ErrMalformedStanza
 			}
 			lastField = canonicalCase(parts[0])
-			lastFieldMultiline = isMultilineField(lastField, isRelease)
+			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
 				stanza[lastField] = parts[1]
 				if parts[1] != "" {

deb/format_test.go

@@ -15,6 +15,10 @@ type ControlFileSuite struct {
 
 var _ = Suite(&ControlFileSuite{})
 
+const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list
+9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+`
+
 const controlFile = `Package: bti
 Binary: bti
 Version: 032-1
@@ -83,15 +87,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadStanza(c *C) {
-	r := NewControlFileReader(s.reader)
+	r := NewControlFileReader(s.reader, false, false)
 
-	stanza1, err := r.ReadStanza(false)
+	stanza1, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza2, err := r.ReadStanza(false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
-	stanza3, err := r.ReadStanza(false)
+	stanza3, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(stanza3, IsNil)
 
@@ -103,27 +107,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) {
 }
 
 func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
-	r := NewControlFileReader(s.reader)
-	stanza, err := r.ReadStanza(false)
+	r := NewControlFileReader(s.reader, false, false)
+	stanza, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	buf := &bytes.Buffer{}
 	w := bufio.NewWriter(buf)
-	err = stanza.Copy().WriteTo(w, true, false)
+	err = stanza.Copy().WriteTo(w, true, false, false)
 	c.Assert(err, IsNil)
 	err = w.Flush()
 	c.Assert(err, IsNil)
 
 	str := buf.String()
 
-	r = NewControlFileReader(buf)
-	stanza2, err := r.ReadStanza(false)
+	r = NewControlFileReader(buf, false, false)
+	stanza2, err := r.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Assert(stanza2, DeepEquals, stanza)
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
+	s.reader = bytes.NewBufferString(installerFile)
+	r := NewControlFileReader(s.reader, false, true)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, false, false, true)
+	c.Assert(err, IsNil)
+	err = w.Flush()
+	c.Assert(err, IsNil)
+
+	r = NewControlFileReader(buf, false, true)
+	stanza2, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Assert(stanza2, DeepEquals, stanza)
+}
+
 func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 	c.Check(canonicalCase("Package"), Equals, "Package")
 	c.Check(canonicalCase("package"), Equals, "Package")
@@ -141,8 +165,8 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 	c.Assert(err, IsNil)
 	defer f.Close()
 
-	r := NewControlFileReader(f)
-	stanza, e := r.ReadStanza(false)
+	r := NewControlFileReader(f, false, false)
+	stanza, e := r.ReadStanza()
 	c.Assert(e, IsNil)
 	c.Assert(len(stanza["Provides"]), Equals, 586929)
 }
@@ -150,9 +174,9 @@ func (s *ControlFileSuite) TestLongFields(c *C) {
 func (s *ControlFileSuite) BenchmarkReadStanza(c *C) {
 	for i := 0; i < c.N; i++ {
 		reader := bytes.NewBufferString(controlFile)
-		r := NewControlFileReader(reader)
+		r := NewControlFileReader(reader, false, false)
 		for {
-			s, e := r.ReadStanza(false)
+			s, e := r.ReadStanza()
 			if s == nil && e == nil {
 				break
 			}

deb/index_files.go

@@ -230,11 +230,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
 	}
 }
 
-func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile {
+func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
 	if arch == ArchitectureSource {
 		udeb = false
 	}
-	key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb)
+	key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string
@@ -244,6 +244,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		} else {
 			if udeb {
 				relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
+			} else if installer {
+				relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
 			} else {
 				relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
 			}
@@ -252,7 +254,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF
 		file = &indexFile{
 			parent:        files,
 			discardable:   false,
-			compressable:  true,
+			compressable:  !installer,
 			signable:      false,
 			acquireByHash: files.acquireByHash,
 			relativePath:  relativePath,

deb/package.go

@@ -1,6 +1,7 @@
 package deb
 
 import (
+	gocontext "context"
 	"encoding/json"
 	"fmt"
 	"path/filepath"
@@ -26,6 +27,8 @@ type Package struct {
 	Provides []string
 	// Hash of files section
 	FilesHash uint64
+	// Is this package a dummy installer package
+	IsInstaller bool
 	// Is this source package
 	IsSource bool
 	// Is this udeb package
@@ -43,9 +46,10 @@ type Package struct {
 
 // Package types
 const (
-	PackageTypeBinary = "deb"
-	PackageTypeUdeb   = "udeb"
-	PackageTypeSource = "source"
+	PackageTypeBinary    = "deb"
+	PackageTypeUdeb      = "udeb"
+	PackageTypeSource    = "source"
+	PackageTypeInstaller = "installer"
 )
 
 // Special arhictectures
@@ -168,6 +172,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package {
 	return p
 }
 
+// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file
+func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) {
+	p := &Package{
+		Name:         "installer",
+		Architecture: architecture,
+		IsInstaller:  true,
+		V06Plus:      true,
+		extra:        &Stanza{},
+		deps:         &PackageDependencies{},
+	}
+
+	files := make(PackageFiles, 0)
+	files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false)
+	if err != nil {
+		return nil, err
+	}
+
+	relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+	for i := range files {
+		files[i].downloadPath = relPath
+
+		url := repo.PackageURL(files[i].DownloadURL()).String()
+		var size int64
+		size, err = d.GetLength(gocontext.TODO(), url)
+		if err != nil {
+			return nil, err
+		}
+
+		files[i].Checksums.Size = size
+	}
+
+	p.UpdateFiles(files)
+	return p, nil
+}
+
 // Key returns unique key identifying package
 func (p *Package) Key(prefix string) []byte {
 	if p.V06Plus {
@@ -509,6 +548,12 @@ func (p *Package) Stanza() (result Stanza) {
 		if len(sha512) > 0 {
 			result["Checksums-Sha512"] = strings.Join(sha512, "")
 		}
+	} else if p.IsInstaller {
+		sha256 := []string{}
+		for _, f := range p.Files() {
+			sha256 = append(sha256, fmt.Sprintf("%s  %s", f.Checksums.SHA256, f.Filename))
+		}
+		result[""] = strings.Join(sha256, "\n")
 	} else {
 		f := p.Files()[0]
 		result["Filename"] = f.DownloadURL()
@@ -563,7 +608,7 @@ func (p *Package) Equals(p2 *Package) bool {
 
 // LinkFromPool links package file from pool to dist's pool location
 func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool,
-	prefix, component string, force bool) error {
+	prefix, distribution, component, architecture string, force bool) error {
 	poolDir, err := p.PoolDirectory()
 	if err != nil {
 		return err
@@ -576,6 +621,9 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
 		}
 
 		relPath := filepath.Join("pool", component, poolDir)
+		if p.IsInstaller {
+			relPath = filepath.Join("dists", distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
+		}
 		publishedDirectory := filepath.Join(prefix, relPath)
 
 		err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)

deb/package_files.go

@@ -88,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool {
 	return files[i].Filename < files[j].Filename
 }
 
-func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) {
+// ParseSumField populates PackageFiles by parsing given input
+func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) {
 	for _, line := range strings.Split(input, "\n") {
 		line = strings.TrimSpace(line)
 		if line == "" {
@@ -96,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 		}
 		parts := strings.Fields(line)
 
-		if len(parts) < 3 {
+		if withSize && len(parts) < 3 || !withSize && len(parts) < 2 {
 			return nil, fmt.Errorf("unparseable hash sum line: %#v", line)
 		}
 
-		size, err := strconv.ParseInt(parts[1], 10, 64)
-		if err != nil {
-			return nil, fmt.Errorf("unable to parse size: %s", err)
+		var size int64
+		var err error
+		if withSize {
+			size, err = strconv.ParseInt(parts[1], 10, 64)
+			if err != nil {
+				return nil, fmt.Errorf("unable to parse size: %s", err)
+			}
 		}
 
-		filename := filepath.Base(parts[len(parts)-1])
+		filename := parts[len(parts)-1]
+		if onlyBasePath {
+			filename = filepath.Base(filename)
+		}
 
 		found := false
 		pos := 0
@@ -133,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che
 func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) {
 	var err error
 
-	files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data })
+	files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}
 
-	files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data })
+	files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true)
 	if err != nil {
 		return nil, err
 	}

deb/package_test.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 
 	"github.com/aptly-dev/aptly/files"
+	"github.com/aptly-dev/aptly/http"
 
 	. "gopkg.in/check.v1"
 )
@@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) {
 	s.stanza = packageStanza.Copy()
 
 	buf := bytes.NewBufferString(sourcePackageMeta)
-	s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false)
+	s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza()
 }
 
 func (s *PackageSuite) TestNewFromPara(c *C) {
@@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) {
 }
 
 func (s *PackageSuite) TestNewUdebFromPara(c *C) {
-	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p := NewUdebPackageFromControlFile(stanza)
 
 	c.Check(p.IsSource, Equals, false)
@@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
 	c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"})
 }
 
+func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
+	repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	downloader := http.NewFakeDownloader()
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
+	downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
+	downloader.ExpectResponse("", "udeb.list")
+	stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza()
+	p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader)
+	c.Check(err, IsNil)
+
+	c.Check(p.IsSource, Equals, false)
+	c.Check(p.IsUdeb, Equals, false)
+	c.Check(p.IsInstaller, Equals, true)
+	c.Check(p.Name, Equals, "installer")
+	c.Check(p.Version, Equals, "")
+	c.Check(p.Architecture, Equals, "i386")
+	c.Check(p.Files(), HasLen, 2)
+	c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs")
+	c.Check(p.Files()[0].Checksums.Size, Equals, int64(14))
+	c.Check(p.Files()[1].Filename, Equals, "./udeb.list")
+	c.Check(p.Files()[1].Checksums.Size, Equals, int64(9))
+}
+
 func (s *PackageSuite) TestNewSourceFromPara(c *C) {
 	p, err := NewSourcePackageFromControlFile(s.sourceStanza)
 
@@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) {
 
 	p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy())
 
-	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false)
+	stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza()
 	p5 := NewUdebPackageFromControlFile(stanza5)
 
 	c.Check(p.GetField("$Source"), Equals, "alien-arena")
@@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
-	err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err := p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false)
 	c.Check(err, IsNil)
 	c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb")
 	c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena")
 
 	p.IsSource = true
-	err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false)
+	err = p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false)
 	c.Check(err, IsNil)
 	c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena")
 }
@@ -490,3 +514,6 @@ Size: 29188
 MD5sum: ae70341c4d96dcded89fa670bcfea31e
 SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba
 SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320`
+
+const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e  ./MANIFEST.udebs
+dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028  ./udeb.list`

deb/publish.go

@@ -569,7 +569,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 		// For all architectures, pregenerate packages/sources files
 		for _, arch := range p.Architectures {
-			indexes.PackageIndex(component, arch, false)
+			indexes.PackageIndex(component, arch, false, false)
 		}
 
 		if progress != nil {
@@ -585,22 +585,17 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				progress.AddBar(1)
 			}
 
-			matches := false
 			for _, arch := range p.Architectures {
 				if pkg.MatchesArchitecture(arch) {
-					matches = true
+					hadUdebs = hadUdebs || pkg.IsUdeb
+					err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Distribution, component, arch, forceOverwrite)
+					if err != nil {
+						return err
+					}
 					break
 				}
 			}
 
-			if matches {
-				hadUdebs = hadUdebs || pkg.IsUdeb
-				err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite)
-				if err != nil {
-					return err
-				}
-			}
-
 			// Start a db batch. If we fill contents data we'll need
 			// to push each path of the package into the database.
 			// We'll want this batched so as to avoid an excessive
@@ -612,7 +607,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 				if pkg.MatchesArchitecture(arch) {
 					var bufWriter *bufio.Writer
 
-					if !p.SkipContents {
+					if !p.SkipContents && !pkg.IsInstaller {
 						key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
 						qualifiedName := []byte(pkg.QualifiedName())
 						contents := pkg.Contents(packagePool, progress)
@@ -629,12 +624,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 						}
 					}
 
-					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
+					bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
 					if err != nil {
 						return err
 					}
 
-					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false)
+					err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller)
 					if err != nil {
 						return err
 					}
@@ -687,7 +682,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 
 			// For all architectures, pregenerate .udeb indexes
 			for _, arch := range p.Architectures {
-				indexes.PackageIndex(component, arch, true)
+				indexes.PackageIndex(component, arch, true, false)
 			}
 		}
 
@@ -710,7 +705,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 					return fmt.Errorf("unable to get ReleaseIndex writer: %s", err)
 				}
 
-				err = release.WriteTo(bufWriter, false, true)
+				err = release.WriteTo(bufWriter, false, true, false)
 				if err != nil {
 					return fmt.Errorf("unable to create Release file: %s", err)
 				}
@@ -791,7 +786,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	err = release.WriteTo(bufWriter, false, true)
+	err = release.WriteTo(bufWriter, false, true, false)
 	if err != nil {
 		return fmt.Errorf("unable to create Release file: %s", err)
 	}

deb/publish_test.go

@@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
 	s.factory.RemoteRepoCollection().Add(repo)
 
@@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
 	c.Assert(err, IsNil)
 
-	cfr := NewControlFileReader(rf)
-	st, err := cfr.ReadStanza(true)
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Origin"], Equals, "ppa squeeze")
@@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
 	pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(pf)
+	cfr = NewControlFileReader(pf, false, false)
 
 	for i := 0; i < 3; i++ {
-		st, err = cfr.ReadStanza(false)
+		st, err = cfr.ReadStanza()
 		c.Assert(err, IsNil)
 
 		c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
 	}
 
-	st, err = cfr.ReadStanza(false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 	c.Assert(st, IsNil)
 
 	drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
 	c.Assert(err, IsNil)
 
-	cfr = NewControlFileReader(drf)
-	st, err = cfr.ReadStanza(true)
+	cfr = NewControlFileReader(drf, true, false)
+	st, err = cfr.ReadStanza()
 	c.Assert(err, IsNil)
 
 	c.Check(st["Archive"], Equals, "squeeze")

deb/remote.go

@@ -68,6 +68,8 @@ type RemoteRepo struct {
 	DownloadSources bool
 	// Should we download .udebs?
 	DownloadUdebs bool
+	// Should we download installer files?
+	DownloadInstaller bool
 	// "Snapshot" of current list of packages
 	packageRefs *PackageRefList
 	// Parsed archived root
@@ -78,16 +80,17 @@ type RemoteRepo struct {
 
 // NewRemoteRepo creates new instance of Debian remote repository with specified params
 func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string,
-	architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) {
+	architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) {
 	result := &RemoteRepo{
-		UUID:            uuid.New(),
-		Name:            name,
-		ArchiveRoot:     archiveRoot,
-		Distribution:    distribution,
-		Components:      components,
-		Architectures:   architectures,
-		DownloadSources: downloadSources,
-		DownloadUdebs:   downloadUdebs,
+		UUID:              uuid.New(),
+		Name:              name,
+		ArchiveRoot:       archiveRoot,
+		Distribution:      distribution,
+		Components:        components,
+		Architectures:     architectures,
+		DownloadSources:   downloadSources,
+		DownloadUdebs:     downloadUdebs,
+		DownloadInstaller: downloadInstaller,
 	}
 
 	err := result.prepare()
@@ -140,6 +143,9 @@ func (repo *RemoteRepo) String() string {
 	if repo.DownloadUdebs {
 		srcFlag += " [udeb]"
 	}
+	if repo.DownloadInstaller {
+		srcFlag += " [installer]"
+	}
 	distribution := repo.Distribution
 	if distribution == "" {
 		distribution = "./"
@@ -243,6 +249,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
 	return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
 }
 
+// InstallerPath returns path of Packages files for given component and
+// architecture
+func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
+	return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
+}
+
 // PackageURL returns URL of package file relative to repository root
 // architecture
 func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
@@ -311,8 +323,8 @@ ok:
 
 	defer release.Close()
 
-	sreader := NewControlFileReader(release)
-	stanza, err := sreader.ReadStanza(true)
+	sreader := NewControlFileReader(release, true, false)
+	stanza, err := sreader.ReadStanza()
 	if err != nil {
 		return err
 	}
@@ -409,7 +421,7 @@ ok:
 }
 
 // DownloadPackageIndexes downloads & parses package index files
-func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory,
+func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
 	ignoreMismatch bool, maxTries int) error {
 	if repo.packageList != nil {
 		panic("packageList != nil")
@@ -420,39 +432,85 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 	packagesPaths := [][]string{}
 
 	if repo.IsFlat() {
-		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
+		packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""})
 		if repo.DownloadSources {
-			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
+			packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""})
 		}
 	} else {
 		for _, component := range repo.Components {
 			for _, architecture := range repo.Architectures {
-				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
+				packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture})
 				if repo.DownloadUdebs {
-					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
+					packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture})
+				}
+				if repo.DownloadInstaller {
+					packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture})
 				}
 			}
 			if repo.DownloadSources {
-				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
+				packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"})
 			}
 		}
 	}
 
 	for _, info := range packagesPaths {
-		path, kind := info[0], info[1]
+		path, kind, component, architecture := info[0], info[1], info[2], info[3]
 		packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
+
+		isInstaller := kind == PackageTypeInstaller
 		if err != nil {
-			return err
+			if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
+				// checking of gpg file is only needed when checksums matches are required.
+				// otherwise there actually has been no candidate found and we can continue
+				if ignoreMismatch {
+					continue
+				}
+
+				// some repos do not have installer hashsum file listed in release file but provide a separate gpg file
+				hashsumPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path}).String()
+				packagesFile, err = http.DownloadTemp(gocontext.TODO(), d, hashsumPath)
+				if err != nil {
+					if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) {
+						// installer files are not available in all components and architectures
+						// so ignore it if not found
+						continue
+					}
+
+					return err
+				}
+
+				if verifier != nil {
+					hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
+					var filesig *os.File
+					filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
+					if err != nil {
+						return err
+					}
+
+					err = verifier.VerifyDetachedSignature(filesig, packagesFile, false)
+					if err != nil {
+						return err
+					}
+
+					_, err = packagesFile.Seek(0, 0)
+				}
+
+				packagesReader = packagesFile
+			}
+
+			if err != nil {
+				return err
+			}
 		}
 		defer packagesFile.Close()
 
 		stat, _ := packagesFile.Stat()
 		progress.InitBar(stat.Size(), true)
 
-		sreader := NewControlFileReader(packagesReader)
+		sreader := NewControlFileReader(packagesReader, false, isInstaller)
 
 		for {
-			stanza, err := sreader.ReadStanza(false)
+			stanza, err := sreader.ReadStanza()
 			if err != nil {
 				return err
 			}
@@ -474,6 +532,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 				if err != nil {
 					return err
 				}
+			} else if kind == PackageTypeInstaller {
+				p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d)
+				if err != nil {
+					return err
+				}
 			}
 			err = repo.packageList.Add(p)
 			if err != nil {

deb/remote_test.go

@@ -88,8 +88,8 @@ type RemoteRepoSuite struct {
 var _ = Suite(&RemoteRepoSuite{})
 
 func (s *RemoteRepoSuite) SetUpTest(c *C) {
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false)
-	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
+	s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false)
 	s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile)
 	s.progress = console.NewProgress()
 	s.db, _ = database.NewOpenDB(c.MkDir())
@@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestInvalidURL(c *C) {
-	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*")
 }
 
@@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) {
 	c.Check(s.flat.Architectures, IsNil)
 	c.Check(s.flat.Components, IsNil)
 
-	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false)
+	flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false)
 	c.Check(flat2.IsFlat(), Equals, true)
 	c.Check(flat2.Distribution, Equals, "./binary/")
 
-	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false)
+	_, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false)
 	c.Check(err, ErrorMatches, "components aren't supported for flat repos")
 }
 
@@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) {
 
 	s.repo.DownloadSources = true
 	s.repo.DownloadUdebs = true
+	s.repo.DownloadInstaller = true
 	s.flat.DownloadSources = true
-	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]")
+	c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]")
 	c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]")
 }
 
@@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) {
 	c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources")
 }
 
+func (s *RemoteRepoSuite) TestInstallerPath(c *C) {
+	c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS")
+}
+
 func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) {
 	c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages")
 }
@@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) {
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*")
 }
 
 func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) {
-	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false)
+	s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false)
 	err := s.repo.Fetch(s.downloader, nil)
 	c.Assert(err, ErrorMatches, "component xyz not available in repo.*")
 }
@@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -347,7 +352,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -391,7 +396,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -416,7 +421,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) {
 	s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404})
 	s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile)
 
-	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1)
+	err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1)
 	c.Assert(err, IsNil)
 	c.Assert(s.downloader.Empty(), Equals, true)
 
@@ -440,7 +445,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -468,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -490,7 +495,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -521,7 +526,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err := s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -567,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -593,7 +598,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) {
 	err = s.flat.Fetch(downloader, nil)
 	c.Assert(err, IsNil)
 
-	err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1)
+	err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1)
 	c.Assert(err, IsNil)
 	c.Assert(downloader.Empty(), Equals, true)
 
@@ -628,7 +633,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) {
 	_, err := s.collection.ByName("yandex")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 	c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists")
 
@@ -646,7 +651,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 	_, err := s.collection.ByUUID("some-uuid")
 	c.Assert(err, ErrorMatches, "*.not found")
 
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Add(repo), IsNil)
 
 	r, err := s.collection.ByUUID(repo.UUID)
@@ -655,7 +660,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	c.Assert(s.collection.Update(repo), IsNil)
 
 	collection := NewRemoteRepoCollection(s.db)
@@ -676,7 +681,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
-	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo)
 
 	count := 0
@@ -698,10 +703,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) {
 }
 
 func (s *RemoteRepoCollectionSuite) TestDrop(c *C) {
-	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo1)
 
-	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false)
+	repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false)
 	s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)

deb/snapshot_test.go

@@ -17,7 +17,7 @@ var _ = Suite(&SnapshotSuite{})
 
 func (s *SnapshotSuite) SetUpTest(c *C) {
 	s.SetUpPackages()
-	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo.packageRefs = s.reflist
 }
 
@@ -116,11 +116,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewSnapshotCollection(s.db)
 	s.SetUpPackages()
 
-	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false)
+	s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	s.repo1.packageRefs = s.reflist
 	s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1)
 
-	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 	s.repo2.packageRefs = s.reflist
 	s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2)
 
@@ -200,7 +200,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) {
 	c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1})
 	c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2})
 
-	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false)
+	repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false)
 
 	c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil))
 }