From 108b0ea22664ec5aa15fbc485c590e2f118a3440 Mon Sep 17 00:00:00 2001 From: Oliver Sauder Date: Thu, 23 Nov 2017 14:22:41 +0100 Subject: [PATCH] Add support to mirror non package installer files --- aptly/interfaces.go | 2 + cmd/mirror_create.go | 4 +- cmd/mirror_update.go | 2 +- cmd/package_show.go | 2 +- completion.d/aptly | 2 +- deb/changes.go | 4 +- deb/deb.go | 8 +- deb/format.go | 53 ++++++--- deb/format_test.go | 50 ++++++--- deb/index_files.go | 8 +- deb/package.go | 56 +++++++++- deb/package_files.go | 28 +++-- deb/package_test.go | 37 +++++- deb/publish.go | 29 ++--- deb/publish_test.go | 16 +-- deb/remote.go | 105 ++++++++++++++---- deb/remote_test.go | 57 +++++----- deb/snapshot_test.go | 8 +- files/public.go | 5 +- http/compression.go | 3 +- http/download.go | 46 ++++++-- http/fake.go | 29 ++++- http/http.go | 11 ++ man/aptly.1 | 4 + system/files/ubuntu-archive-keyring.gpg | Bin 0 -> 12335 bytes system/lib.py | 1 + system/t03_help/MirrorCreateHelpTest_gold | 1 + system/t03_help/MirrorCreateTest_gold | 1 + system/t03_help/WrongFlagTest_gold | 1 + system/t04_mirror/UpdateMirror23Test_gold | 33 ++++++ system/t04_mirror/UpdateMirror24Test_gold | 61 ++++++++++ system/t04_mirror/update.py | 32 ++++++ system/t06_publish/PublishSnapshot38Test_gold | 13 +++ .../PublishSnapshot38Test_installer_s390x | 12 ++ system/t06_publish/snapshot.py | 22 ++++ 35 files changed, 593 insertions(+), 153 deletions(-) create mode 100644 system/files/ubuntu-archive-keyring.gpg create mode 100644 system/t04_mirror/UpdateMirror23Test_gold create mode 100644 system/t04_mirror/UpdateMirror24Test_gold create mode 100644 system/t06_publish/PublishSnapshot38Test_gold create mode 100644 system/t06_publish/PublishSnapshot38Test_installer_s390x diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 53e27e04..29a30566 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -130,6 +130,8 @@ type Downloader interface { DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error // GetProgress returns Progress object GetProgress() Progress + // GetLength of given url + GetLength(ctx context.Context, url string) (int64, error) } // ChecksumStorage is stores checksums in some (persistent) storage diff --git a/cmd/mirror_create.go b/cmd/mirror_create.go index a45cbf82..7477bf34 100644 --- a/cmd/mirror_create.go +++ b/cmd/mirror_create.go @@ -19,6 +19,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources") downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool) + downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool) var ( mirrorName, archiveURL, distribution string @@ -36,7 +37,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error { } repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), - downloadSources, downloadUdebs) + downloadSources, downloadUdebs, downloadInstaller) if err != nil { return fmt.Errorf("unable to create mirror: %s", err) } @@ -94,6 +95,7 @@ Example: } cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") + cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") cmd.Flag.String("filter", "", "filter packages in mirror") diff --git a/cmd/mirror_update.go b/cmd/mirror_update.go index c087aeed..8fac46f0 100644 --- a/cmd/mirror_update.go +++ b/cmd/mirror_update.go @@ -54,7 +54,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error { } context.Progress().Printf("Downloading & parsing package files...\n") - err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), context.CollectionFactory(), ignoreMismatch, maxTries) + err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, context.CollectionFactory(), ignoreMismatch, maxTries) if err != nil { return fmt.Errorf("unable to update: %s", err) } diff --git a/cmd/package_show.go b/cmd/package_show.go index 643e75fa..d1e50332 100644 --- a/cmd/package_show.go +++ b/cmd/package_show.go @@ -79,7 +79,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error { result := q.Query(context.CollectionFactory().PackageCollection()) err = result.ForEach(func(p *deb.Package) error { - p.Stanza().WriteTo(w, p.IsSource, false) + p.Stanza().WriteTo(w, p.IsSource, false, false) w.Flush() fmt.Printf("\n") diff --git a/completion.d/aptly b/completion.d/aptly index f2897633..6e573750 100644 --- a/completion.d/aptly +++ b/completion.d/aptly @@ -156,7 +156,7 @@ _aptly() "create") if [[ $numargs -eq 0 ]]; then if [[ "$cur" == -* ]]; then - COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-sources -with-udebs" -- ${cur})) + COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur})) return 0 fi fi diff --git a/deb/changes.go b/deb/changes.go index de4c5de6..7794b3a3 100644 --- a/deb/changes.go +++ b/deb/changes.go @@ -95,8 +95,8 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier text = input } - reader := NewControlFileReader(text) - c.Stanza, err = reader.ReadStanza(false) + reader := NewControlFileReader(text, false, false) + c.Stanza, err = reader.ReadStanza() if err != nil { return err } diff --git a/deb/deb.go b/deb/deb.go index 3be055ec..169e3c43 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -89,8 +89,8 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) { } if tarHeader.Name == "./control" || tarHeader.Name == "control" { - reader := NewControlFileReader(untar) - stanza, err := reader.ReadStanza(false) + reader := NewControlFileReader(untar, false, false) + stanza, err := reader.ReadStanza() if err != nil { return nil, err } @@ -129,8 +129,8 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error text = file } - reader := NewControlFileReader(text) - stanza, err := reader.ReadStanza(false) + reader := NewControlFileReader(text, false, false) + stanza, err := reader.ReadStanza() if err != nil { return nil, err } diff --git a/deb/format.go b/deb/format.go index 82dd2210..3d02ab46 100644 --- a/deb/format.go +++ b/deb/format.go @@ -88,6 +88,9 @@ var ( "Directory", "Files", } + canonicalOrderInstaller = []string{ + "", + } ) // Copy returns copy of Stanza @@ -101,6 +104,9 @@ func (s Stanza) Copy() (result Stanza) { func isMultilineField(field string, isRelease bool) bool { switch field { + // file without a section + case "": + return true case "Description": return true case "Files": @@ -132,21 +138,26 @@ func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error if !isMultilineField(field, isRelease) { _, err = w.WriteString(field + ": " + value + "\n") } else { - if !strings.HasSuffix(value, "\n") { + if field != "" && !strings.HasSuffix(value, "\n") { value = value + "\n" } - if field != "Description" { + if field != "Description" && field != "" { value = "\n" + value } - _, err = w.WriteString(field + ":" + value) + + if field != "" { + _, err = w.WriteString(field + ":" + value) + } else { + _, err = w.WriteString(value) + } } return } // WriteTo saves stanza back to stream, modifying itself on the fly -func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { +func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease, isInstaller bool) error { canonicalOrder := canonicalOrderBinary if isSource { canonicalOrder = canonicalOrderSource @@ -154,6 +165,9 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { if isRelease { canonicalOrder = canonicalOrderRelease } + if isInstaller { + canonicalOrder = canonicalOrderInstaller + } for _, field := range canonicalOrder { value, ok := s[field] @@ -166,10 +180,13 @@ func (s Stanza) WriteTo(w *bufio.Writer, isSource, isRelease bool) error { } } - for field, value := range s { - err := writeField(w, field, value, isRelease) - if err != nil { - return err + // no extra fields in installer + if !isInstaller { + for field, value := range s { + err := writeField(w, field, value, isRelease) + if err != nil { + return err + } } } @@ -212,22 +229,28 @@ func canonicalCase(field string) string { // ControlFileReader implements reading of control files stanza by stanza type ControlFileReader struct { - scanner *bufio.Scanner + scanner *bufio.Scanner + isRelease bool + isInstaller bool } // NewControlFileReader creates ControlFileReader, it wraps with buffering -func NewControlFileReader(r io.Reader) *ControlFileReader { +func NewControlFileReader(r io.Reader, isRelease, isInstaller bool) *ControlFileReader { scnr := bufio.NewScanner(bufio.NewReaderSize(r, 32768)) scnr.Buffer(nil, MaxFieldSize) - return &ControlFileReader{scanner: scnr} + return &ControlFileReader{ + scanner: scnr, + isRelease: isRelease, + isInstaller: isInstaller, + } } // ReadStanza reeads one stanza from control file -func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { +func (c *ControlFileReader) ReadStanza() (Stanza, error) { stanza := make(Stanza, 32) lastField := "" - lastFieldMultiline := false + lastFieldMultiline := c.isInstaller for c.scanner.Scan() { line := c.scanner.Text() @@ -240,7 +263,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { continue } - if line[0] == ' ' || line[0] == '\t' { + if line[0] == ' ' || line[0] == '\t' || c.isInstaller { if lastFieldMultiline { stanza[lastField] += line + "\n" } else { @@ -252,7 +275,7 @@ func (c *ControlFileReader) ReadStanza(isRelease bool) (Stanza, error) { return nil, ErrMalformedStanza } lastField = canonicalCase(parts[0]) - lastFieldMultiline = isMultilineField(lastField, isRelease) + lastFieldMultiline = isMultilineField(lastField, c.isRelease) if lastFieldMultiline { stanza[lastField] = parts[1] if parts[1] != "" { diff --git a/deb/format_test.go b/deb/format_test.go index 38cbec57..2c32b033 100644 --- a/deb/format_test.go +++ b/deb/format_test.go @@ -15,6 +15,10 @@ type ControlFileSuite struct { var _ = Suite(&ControlFileSuite{}) +const installerFile = `dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list +9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs +` + const controlFile = `Package: bti Binary: bti Version: 032-1 @@ -83,15 +87,15 @@ func (s *ControlFileSuite) SetUpTest(c *C) { } func (s *ControlFileSuite) TestReadStanza(c *C) { - r := NewControlFileReader(s.reader) + r := NewControlFileReader(s.reader, false, false) - stanza1, err := r.ReadStanza(false) + stanza1, err := r.ReadStanza() c.Assert(err, IsNil) - stanza2, err := r.ReadStanza(false) + stanza2, err := r.ReadStanza() c.Assert(err, IsNil) - stanza3, err := r.ReadStanza(false) + stanza3, err := r.ReadStanza() c.Assert(err, IsNil) c.Assert(stanza3, IsNil) @@ -103,27 +107,47 @@ func (s *ControlFileSuite) TestReadStanza(c *C) { } func (s *ControlFileSuite) TestReadWriteStanza(c *C) { - r := NewControlFileReader(s.reader) - stanza, err := r.ReadStanza(false) + r := NewControlFileReader(s.reader, false, false) + stanza, err := r.ReadStanza() c.Assert(err, IsNil) buf := &bytes.Buffer{} w := bufio.NewWriter(buf) - err = stanza.Copy().WriteTo(w, true, false) + err = stanza.Copy().WriteTo(w, true, false, false) c.Assert(err, IsNil) err = w.Flush() c.Assert(err, IsNil) str := buf.String() - r = NewControlFileReader(buf) - stanza2, err := r.ReadStanza(false) + r = NewControlFileReader(buf, false, false) + stanza2, err := r.ReadStanza() c.Assert(err, IsNil) c.Assert(stanza2, DeepEquals, stanza) c.Assert(strings.HasPrefix(str, "Package: "), Equals, true) } +func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) { + s.reader = bytes.NewBufferString(installerFile) + r := NewControlFileReader(s.reader, false, true) + stanza, err := r.ReadStanza() + c.Assert(err, IsNil) + + buf := &bytes.Buffer{} + w := bufio.NewWriter(buf) + err = stanza.Copy().WriteTo(w, false, false, true) + c.Assert(err, IsNil) + err = w.Flush() + c.Assert(err, IsNil) + + r = NewControlFileReader(buf, false, true) + stanza2, err := r.ReadStanza() + c.Assert(err, IsNil) + + c.Assert(stanza2, DeepEquals, stanza) +} + func (s *ControlFileSuite) TestCanonicalCase(c *C) { c.Check(canonicalCase("Package"), Equals, "Package") c.Check(canonicalCase("package"), Equals, "Package") @@ -141,8 +165,8 @@ func (s *ControlFileSuite) TestLongFields(c *C) { c.Assert(err, IsNil) defer f.Close() - r := NewControlFileReader(f) - stanza, e := r.ReadStanza(false) + r := NewControlFileReader(f, false, false) + stanza, e := r.ReadStanza() c.Assert(e, IsNil) c.Assert(len(stanza["Provides"]), Equals, 586929) } @@ -150,9 +174,9 @@ func (s *ControlFileSuite) TestLongFields(c *C) { func (s *ControlFileSuite) BenchmarkReadStanza(c *C) { for i := 0; i < c.N; i++ { reader := bytes.NewBufferString(controlFile) - r := NewControlFileReader(reader) + r := NewControlFileReader(reader, false, false) for { - s, e := r.ReadStanza(false) + s, e := r.ReadStanza() if s == nil && e == nil { break } diff --git a/deb/index_files.go b/deb/index_files.go index 713e6c95..c4805ce5 100644 --- a/deb/index_files.go +++ b/deb/index_files.go @@ -230,11 +230,11 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s } } -func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexFile { +func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile { if arch == ArchitectureSource { udeb = false } - key := fmt.Sprintf("pi-%s-%s-%v", component, arch, udeb) + key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer) file, ok := files.indexes[key] if !ok { var relativePath string @@ -244,6 +244,8 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF } else { if udeb { relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages") + } else if installer { + relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS") } else { relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages") } @@ -252,7 +254,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool) *indexF file = &indexFile{ parent: files, discardable: false, - compressable: true, + compressable: !installer, signable: false, acquireByHash: files.acquireByHash, relativePath: relativePath, diff --git a/deb/package.go b/deb/package.go index 801d7a2d..ae3e5063 100644 --- a/deb/package.go +++ b/deb/package.go @@ -1,6 +1,7 @@ package deb import ( + gocontext "context" "encoding/json" "fmt" "path/filepath" @@ -26,6 +27,8 @@ type Package struct { Provides []string // Hash of files section FilesHash uint64 + // Is this package a dummy installer package + IsInstaller bool // Is this source package IsSource bool // Is this udeb package @@ -43,9 +46,10 @@ type Package struct { // Package types const ( - PackageTypeBinary = "deb" - PackageTypeUdeb = "udeb" - PackageTypeSource = "source" + PackageTypeBinary = "deb" + PackageTypeUdeb = "udeb" + PackageTypeSource = "source" + PackageTypeInstaller = "installer" ) // Special arhictectures @@ -168,6 +172,41 @@ func NewUdebPackageFromControlFile(input Stanza) *Package { return p } +// NewInstallerPackageFromControlFile creates a dummy installer Package from parsed hash sum file +func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, component, architecture string, d aptly.Downloader) (*Package, error) { + p := &Package{ + Name: "installer", + Architecture: architecture, + IsInstaller: true, + V06Plus: true, + extra: &Stanza{}, + deps: &PackageDependencies{}, + } + + files := make(PackageFiles, 0) + files, err := files.ParseSumField(input[""], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, false, false) + if err != nil { + return nil, err + } + + relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images") + for i := range files { + files[i].downloadPath = relPath + + url := repo.PackageURL(files[i].DownloadURL()).String() + var size int64 + size, err = d.GetLength(gocontext.TODO(), url) + if err != nil { + return nil, err + } + + files[i].Checksums.Size = size + } + + p.UpdateFiles(files) + return p, nil +} + // Key returns unique key identifying package func (p *Package) Key(prefix string) []byte { if p.V06Plus { @@ -509,6 +548,12 @@ func (p *Package) Stanza() (result Stanza) { if len(sha512) > 0 { result["Checksums-Sha512"] = strings.Join(sha512, "") } + } else if p.IsInstaller { + sha256 := []string{} + for _, f := range p.Files() { + sha256 = append(sha256, fmt.Sprintf("%s %s", f.Checksums.SHA256, f.Filename)) + } + result[""] = strings.Join(sha256, "\n") } else { f := p.Files()[0] result["Filename"] = f.DownloadURL() @@ -563,7 +608,7 @@ func (p *Package) Equals(p2 *Package) bool { // LinkFromPool links package file from pool to dist's pool location func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packagePool aptly.PackagePool, - prefix, component string, force bool) error { + prefix, distribution, component, architecture string, force bool) error { poolDir, err := p.PoolDirectory() if err != nil { return err @@ -576,6 +621,9 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP } relPath := filepath.Join("pool", component, poolDir) + if p.IsInstaller { + relPath = filepath.Join("dists", distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images") + } publishedDirectory := filepath.Join(prefix, relPath) err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force) diff --git a/deb/package_files.go b/deb/package_files.go index 61f07ac6..ae208b69 100644 --- a/deb/package_files.go +++ b/deb/package_files.go @@ -88,7 +88,8 @@ func (files PackageFiles) Less(i, j int) bool { return files[i].Filename < files[j].Filename } -func (files PackageFiles) parseSumField(input string, setter func(sum *utils.ChecksumInfo, data string)) (PackageFiles, error) { +// ParseSumField populates PackageFiles by parsing given input +func (files PackageFiles) ParseSumField(input string, setter func(sum *utils.ChecksumInfo, data string), withSize bool, onlyBasePath bool) (PackageFiles, error) { for _, line := range strings.Split(input, "\n") { line = strings.TrimSpace(line) if line == "" { @@ -96,16 +97,23 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che } parts := strings.Fields(line) - if len(parts) < 3 { + if withSize && len(parts) < 3 || !withSize && len(parts) < 2 { return nil, fmt.Errorf("unparseable hash sum line: %#v", line) } - size, err := strconv.ParseInt(parts[1], 10, 64) - if err != nil { - return nil, fmt.Errorf("unable to parse size: %s", err) + var size int64 + var err error + if withSize { + size, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("unable to parse size: %s", err) + } } - filename := filepath.Base(parts[len(parts)-1]) + filename := parts[len(parts)-1] + if onlyBasePath { + filename = filepath.Base(filename) + } found := false pos := 0 @@ -133,22 +141,22 @@ func (files PackageFiles) parseSumField(input string, setter func(sum *utils.Che func (files PackageFiles) ParseSumFields(stanza Stanza) (PackageFiles, error) { var err error - files, err = files.parseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }) + files, err = files.ParseSumField(stanza["Files"], func(sum *utils.ChecksumInfo, data string) { sum.MD5 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha1"], func(sum *utils.ChecksumInfo, data string) { sum.SHA1 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha256"], func(sum *utils.ChecksumInfo, data string) { sum.SHA256 = data }, true, true) if err != nil { return nil, err } - files, err = files.parseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }) + files, err = files.ParseSumField(stanza["Checksums-Sha512"], func(sum *utils.ChecksumInfo, data string) { sum.SHA512 = data }, true, true) if err != nil { return nil, err } diff --git a/deb/package_test.go b/deb/package_test.go index af6d4b29..704e083a 100644 --- a/deb/package_test.go +++ b/deb/package_test.go @@ -7,6 +7,7 @@ import ( "regexp" "github.com/aptly-dev/aptly/files" + "github.com/aptly-dev/aptly/http" . "gopkg.in/check.v1" ) @@ -22,7 +23,7 @@ func (s *PackageSuite) SetUpTest(c *C) { s.stanza = packageStanza.Copy() buf := bytes.NewBufferString(sourcePackageMeta) - s.sourceStanza, _ = NewControlFileReader(buf).ReadStanza(false) + s.sourceStanza, _ = NewControlFileReader(buf, false, false).ReadStanza() } func (s *PackageSuite) TestNewFromPara(c *C) { @@ -43,7 +44,7 @@ func (s *PackageSuite) TestNewFromPara(c *C) { } func (s *PackageSuite) TestNewUdebFromPara(c *C) { - stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false) + stanza, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza() p := NewUdebPackageFromControlFile(stanza) c.Check(p.IsSource, Equals, false) @@ -57,6 +58,29 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) { c.Check(p.deps.Depends, DeepEquals, []string{"libc6-udeb (>= 2.13)"}) } +func (s *PackageSuite) TestNewInstallerFromPara(c *C) { + repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + downloader := http.NewFakeDownloader() + downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs") + downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list") + downloader.ExpectResponse("", "udeb.list") + stanza, _ := NewControlFileReader(bytes.NewBufferString(installerPackageMeta), false, true).ReadStanza() + p, err := NewInstallerPackageFromControlFile(stanza, repo, "main", "i386", downloader) + c.Check(err, IsNil) + + c.Check(p.IsSource, Equals, false) + c.Check(p.IsUdeb, Equals, false) + c.Check(p.IsInstaller, Equals, true) + c.Check(p.Name, Equals, "installer") + c.Check(p.Version, Equals, "") + c.Check(p.Architecture, Equals, "i386") + c.Check(p.Files(), HasLen, 2) + c.Check(p.Files()[0].Filename, Equals, "./MANIFEST.udebs") + c.Check(p.Files()[0].Checksums.Size, Equals, int64(14)) + c.Check(p.Files()[1].Filename, Equals, "./udeb.list") + c.Check(p.Files()[1].Checksums.Size, Equals, int64(9)) +} + func (s *PackageSuite) TestNewSourceFromPara(c *C) { p, err := NewSourcePackageFromControlFile(s.sourceStanza) @@ -156,7 +180,7 @@ func (s *PackageSuite) TestGetField(c *C) { p4, _ := NewSourcePackageFromControlFile(s.sourceStanza.Copy()) - stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta)).ReadStanza(false) + stanza5, _ := NewControlFileReader(bytes.NewBufferString(udebPackageMeta), false, false).ReadStanza() p5 := NewUdebPackageFromControlFile(stanza5) c.Check(p.GetField("$Source"), Equals, "alien-arena") @@ -372,13 +396,13 @@ func (s *PackageSuite) TestLinkFromPool(c *C) { p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs) - err := p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) + err := p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false) c.Check(err, IsNil) c.Check(p.Files()[0].Filename, Equals, "alien-arena-common_7.40-2_i386.deb") c.Check(p.Files()[0].downloadPath, Equals, "pool/non-free/a/alien-arena") p.IsSource = true - err = p.LinkFromPool(publishedStorage, packagePool, "", "non-free", false) + err = p.LinkFromPool(publishedStorage, packagePool, "", "jessie", "non-free", "i386", false) c.Check(err, IsNil) c.Check(p.Extra()["Directory"], Equals, "pool/non-free/a/alien-arena") } @@ -490,3 +514,6 @@ Size: 29188 MD5sum: ae70341c4d96dcded89fa670bcfea31e SHA1: 9532ae4226a85805189a671ee0283f719d48a5ba SHA256: bbb3a2cb07f741c3995b6d4bb08d772d83582b93a0236d4ea7736bc0370fc320` + +const installerPackageMeta = `9d8bb14044dee520f4706ab197dfff10e9e39ecb3c1a402331712154e8284b2e ./MANIFEST.udebs +dab96042d8e25e0f6bbb8d7c5bd78543afb5eb31a4a8b122ece68ab197228028 ./udeb.list` diff --git a/deb/publish.go b/deb/publish.go index 13152ef5..6776bf46 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -569,7 +569,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP // For all architectures, pregenerate packages/sources files for _, arch := range p.Architectures { - indexes.PackageIndex(component, arch, false) + indexes.PackageIndex(component, arch, false, false) } if progress != nil { @@ -585,22 +585,17 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP progress.AddBar(1) } - matches := false for _, arch := range p.Architectures { if pkg.MatchesArchitecture(arch) { - matches = true + hadUdebs = hadUdebs || pkg.IsUdeb + err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, p.Distribution, component, arch, forceOverwrite) + if err != nil { + return err + } break } } - if matches { - hadUdebs = hadUdebs || pkg.IsUdeb - err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, component, forceOverwrite) - if err != nil { - return err - } - } - // Start a db batch. If we fill contents data we'll need // to push each path of the package into the database. // We'll want this batched so as to avoid an excessive @@ -612,7 +607,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP if pkg.MatchesArchitecture(arch) { var bufWriter *bufio.Writer - if !p.SkipContents { + if !p.SkipContents && !pkg.IsInstaller { key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb) qualifiedName := []byte(pkg.QualifiedName()) contents := pkg.Contents(packagePool, progress) @@ -629,12 +624,12 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP } } - bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter() + bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter() if err != nil { return err } - err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false) + err = pkg.Stanza().WriteTo(bufWriter, pkg.IsSource, false, pkg.IsInstaller) if err != nil { return err } @@ -687,7 +682,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP // For all architectures, pregenerate .udeb indexes for _, arch := range p.Architectures { - indexes.PackageIndex(component, arch, true) + indexes.PackageIndex(component, arch, true, false) } } @@ -710,7 +705,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP return fmt.Errorf("unable to get ReleaseIndex writer: %s", err) } - err = release.WriteTo(bufWriter, false, true) + err = release.WriteTo(bufWriter, false, true, false) if err != nil { return fmt.Errorf("unable to create Release file: %s", err) } @@ -791,7 +786,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP return err } - err = release.WriteTo(bufWriter, false, true) + err = release.WriteTo(bufWriter, false, true, false) if err != nil { return fmt.Errorf("unable to create Release file: %s", err) } diff --git a/deb/publish_test.go b/deb/publish_test.go index 15df26f1..371823e8 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -113,7 +113,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) { s.reflist = NewPackageRefListFromPackageList(s.list) - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) repo.packageRefs = s.reflist s.factory.RemoteRepoCollection().Add(repo) @@ -314,8 +314,8 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release")) c.Assert(err, IsNil) - cfr := NewControlFileReader(rf) - st, err := cfr.ReadStanza(true) + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Origin"], Equals, "ppa squeeze") @@ -325,24 +325,24 @@ func (s *PublishedRepoSuite) TestPublish(c *C) { pf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages")) c.Assert(err, IsNil) - cfr = NewControlFileReader(pf) + cfr = NewControlFileReader(pf, false, false) for i := 0; i < 3; i++ { - st, err = cfr.ReadStanza(false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Filename"], Equals, "pool/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb") } - st, err = cfr.ReadStanza(false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Assert(st, IsNil) drf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release")) c.Assert(err, IsNil) - cfr = NewControlFileReader(drf) - st, err = cfr.ReadStanza(true) + cfr = NewControlFileReader(drf, true, false) + st, err = cfr.ReadStanza() c.Assert(err, IsNil) c.Check(st["Archive"], Equals, "squeeze") diff --git a/deb/remote.go b/deb/remote.go index 167d45f1..2a862858 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -68,6 +68,8 @@ type RemoteRepo struct { DownloadSources bool // Should we download .udebs? DownloadUdebs bool + // Should we download installer files? + DownloadInstaller bool // "Snapshot" of current list of packages packageRefs *PackageRefList // Parsed archived root @@ -78,16 +80,17 @@ type RemoteRepo struct { // NewRemoteRepo creates new instance of Debian remote repository with specified params func NewRemoteRepo(name string, archiveRoot string, distribution string, components []string, - architectures []string, downloadSources bool, downloadUdebs bool) (*RemoteRepo, error) { + architectures []string, downloadSources bool, downloadUdebs bool, downloadInstaller bool) (*RemoteRepo, error) { result := &RemoteRepo{ - UUID: uuid.New(), - Name: name, - ArchiveRoot: archiveRoot, - Distribution: distribution, - Components: components, - Architectures: architectures, - DownloadSources: downloadSources, - DownloadUdebs: downloadUdebs, + UUID: uuid.New(), + Name: name, + ArchiveRoot: archiveRoot, + Distribution: distribution, + Components: components, + Architectures: architectures, + DownloadSources: downloadSources, + DownloadUdebs: downloadUdebs, + DownloadInstaller: downloadInstaller, } err := result.prepare() @@ -140,6 +143,9 @@ func (repo *RemoteRepo) String() string { if repo.DownloadUdebs { srcFlag += " [udeb]" } + if repo.DownloadInstaller { + srcFlag += " [installer]" + } distribution := repo.Distribution if distribution == "" { distribution = "./" @@ -243,6 +249,12 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string { return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture) } +// InstallerPath returns path of Packages files for given component and +// architecture +func (repo *RemoteRepo) InstallerPath(component string, architecture string) string { + return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture) +} + // PackageURL returns URL of package file relative to repository root // architecture func (repo *RemoteRepo) PackageURL(filename string) *url.URL { @@ -311,8 +323,8 @@ ok: defer release.Close() - sreader := NewControlFileReader(release) - stanza, err := sreader.ReadStanza(true) + sreader := NewControlFileReader(release, true, false) + stanza, err := sreader.ReadStanza() if err != nil { return err } @@ -409,7 +421,7 @@ ok: } // DownloadPackageIndexes downloads & parses package index files -func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, collectionFactory *CollectionFactory, +func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory, ignoreMismatch bool, maxTries int) error { if repo.packageList != nil { panic("packageList != nil") @@ -420,39 +432,85 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. packagesPaths := [][]string{} if repo.IsFlat() { - packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary}) + packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary, "", ""}) if repo.DownloadSources { - packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource}) + packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource, "", ""}) } } else { for _, component := range repo.Components { for _, architecture := range repo.Architectures { - packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary}) + packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary, component, architecture}) if repo.DownloadUdebs { - packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb}) + packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb, component, architecture}) + } + if repo.DownloadInstaller { + packagesPaths = append(packagesPaths, []string{repo.InstallerPath(component, architecture), PackageTypeInstaller, component, architecture}) } } if repo.DownloadSources { - packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource}) + packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource, component, "source"}) } } } for _, info := range packagesPaths { - path, kind := info[0], info[1] + path, kind, component, architecture := info[0], info[1], info[2], info[3] packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries) + + isInstaller := kind == PackageTypeInstaller if err != nil { - return err + if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok { + // checking of gpg file is only needed when checksums matches are required. + // otherwise there actually has been no candidate found and we can continue + if ignoreMismatch { + continue + } + + // some repos do not have installer hashsum file listed in release file but provide a separate gpg file + hashsumPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path}).String() + packagesFile, err = http.DownloadTemp(gocontext.TODO(), d, hashsumPath) + if err != nil { + if herr, ok := err.(*http.Error); ok && (herr.Code == 404 || herr.Code == 403) { + // installer files are not available in all components and architectures + // so ignore it if not found + continue + } + + return err + } + + if verifier != nil { + hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String() + var filesig *os.File + filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath) + if err != nil { + return err + } + + err = verifier.VerifyDetachedSignature(filesig, packagesFile, false) + if err != nil { + return err + } + + _, err = packagesFile.Seek(0, 0) + } + + packagesReader = packagesFile + } + + if err != nil { + return err + } } defer packagesFile.Close() stat, _ := packagesFile.Stat() progress.InitBar(stat.Size(), true) - sreader := NewControlFileReader(packagesReader) + sreader := NewControlFileReader(packagesReader, false, isInstaller) for { - stanza, err := sreader.ReadStanza(false) + stanza, err := sreader.ReadStanza() if err != nil { return err } @@ -474,6 +532,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. if err != nil { return err } + } else if kind == PackageTypeInstaller { + p, err = NewInstallerPackageFromControlFile(stanza, repo, component, architecture, d) + if err != nil { + return err + } } err = repo.packageList.Add(p) if err != nil { diff --git a/deb/remote_test.go b/deb/remote_test.go index 568ec5af..8f90e0ce 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -88,8 +88,8 @@ type RemoteRepoSuite struct { var _ = Suite(&RemoteRepoSuite{}) func (s *RemoteRepoSuite) SetUpTest(c *C) { - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false) - s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian", "squeeze", []string{"main"}, []string{}, false, false, false) + s.flat, _ = NewRemoteRepo("exp42", "http://repos.express42.com/virool/precise/", "./", []string{}, []string{}, false, false, false) s.downloader = http.NewFakeDownloader().ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/Release", exampleReleaseFile) s.progress = console.NewProgress() s.db, _ = database.NewOpenDB(c.MkDir()) @@ -106,7 +106,7 @@ func (s *RemoteRepoSuite) TearDownTest(c *C) { } func (s *RemoteRepoSuite) TestInvalidURL(c *C) { - _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false) + _, err := NewRemoteRepo("s", "http://lolo%2", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(err, ErrorMatches, ".*(hexadecimal escape in host|percent-encoded characters in host|invalid URL escape).*") } @@ -116,11 +116,11 @@ func (s *RemoteRepoSuite) TestFlatCreation(c *C) { c.Check(s.flat.Architectures, IsNil) c.Check(s.flat.Components, IsNil) - flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false) + flat2, _ := NewRemoteRepo("flat2", "http://pkg.jenkins-ci.org/debian-stable", "binary/", []string{}, []string{}, false, false, false) c.Check(flat2.IsFlat(), Equals, true) c.Check(flat2.Distribution, Equals, "./binary/") - _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false) + _, err := NewRemoteRepo("fl", "http://some.repo/", "./", []string{"main"}, []string{}, false, false, false) c.Check(err, ErrorMatches, "components aren't supported for flat repos") } @@ -130,8 +130,9 @@ func (s *RemoteRepoSuite) TestString(c *C) { s.repo.DownloadSources = true s.repo.DownloadUdebs = true + s.repo.DownloadInstaller = true s.flat.DownloadSources = true - c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb]") + c.Check(s.repo.String(), Equals, "[yandex]: http://mirror.yandex.ru/debian/ squeeze [src] [udeb] [installer]") c.Check(s.flat.String(), Equals, "[exp42]: http://repos.express42.com/virool/precise/ ./ [src]") } @@ -176,6 +177,10 @@ func (s *RemoteRepoSuite) TestSourcesPath(c *C) { c.Assert(s.repo.SourcesPath("main"), Equals, "main/source/Sources") } +func (s *RemoteRepoSuite) TestInstallerPath(c *C) { + c.Assert(s.repo.InstallerPath("main", "amd64"), Equals, "main/installer-amd64/current/images/SHA256SUMS") +} + func (s *RemoteRepoSuite) TestFlatBinaryPath(c *C) { c.Assert(s.flat.FlatBinaryPath(), Equals, "Packages") } @@ -230,13 +235,13 @@ func (s *RemoteRepoSuite) TestFetchNullVerifier2(c *C) { } func (s *RemoteRepoSuite) TestFetchWrongArchitecture(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{"xyz"}, false, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "architecture xyz not available in repo.*") } func (s *RemoteRepoSuite) TestFetchWrongComponent(c *C) { - s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false) + s.repo, _ = NewRemoteRepo("s", "http://mirror.yandex.ru/debian/", "squeeze", []string{"xyz"}, []string{"i386"}, false, false, false) err := s.repo.Fetch(s.downloader, nil) c.Assert(err, ErrorMatches, "component xyz not available in repo.*") } @@ -271,7 +276,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -298,7 +303,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -319,7 +324,7 @@ func (s *RemoteRepoSuite) TestDownload(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/binary-i386/Packages", examplePackagesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -347,7 +352,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -391,7 +396,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -416,7 +421,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSources(c *C) { s.downloader.ExpectError("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources.gz", &http.Error{Code: 404}) s.downloader.ExpectResponse("http://mirror.yandex.ru/debian/dists/squeeze/main/source/Sources", exampleSourcesFile) - err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, s.collectionFactory, false, 1) + err = s.repo.DownloadPackageIndexes(s.progress, s.downloader, nil, s.collectionFactory, false, 1) c.Assert(err, IsNil) c.Assert(s.downloader.Empty(), Equals, true) @@ -440,7 +445,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err := s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -468,7 +473,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -490,7 +495,7 @@ func (s *RemoteRepoSuite) TestDownloadFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -521,7 +526,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err := s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -567,7 +572,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -593,7 +598,7 @@ func (s *RemoteRepoSuite) TestDownloadWithSourcesFlat(c *C) { err = s.flat.Fetch(downloader, nil) c.Assert(err, IsNil) - err = s.flat.DownloadPackageIndexes(s.progress, downloader, s.collectionFactory, true, 1) + err = s.flat.DownloadPackageIndexes(s.progress, downloader, nil, s.collectionFactory, true, 1) c.Assert(err, IsNil) c.Assert(downloader.Empty(), Equals, true) @@ -628,7 +633,7 @@ func (s *RemoteRepoCollectionSuite) TestAddByName(c *C) { _, err := s.collection.ByName("yandex") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Add(repo), IsNil) c.Assert(s.collection.Add(repo), ErrorMatches, ".*already exists") @@ -646,7 +651,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { _, err := s.collection.ByUUID("some-uuid") c.Assert(err, ErrorMatches, "*.not found") - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Add(repo), IsNil) r, err := s.collection.ByUUID(repo.UUID) @@ -655,7 +660,7 @@ func (s *RemoteRepoCollectionSuite) TestByUUID(c *C) { } func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) c.Assert(s.collection.Update(repo), IsNil) collection := NewRemoteRepoCollection(s.db) @@ -676,7 +681,7 @@ func (s *RemoteRepoCollectionSuite) TestUpdateLoadComplete(c *C) { } func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { - repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo) count := 0 @@ -698,10 +703,10 @@ func (s *RemoteRepoCollectionSuite) TestForEachAndLen(c *C) { } func (s *RemoteRepoCollectionSuite) TestDrop(c *C) { - repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + repo1, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo1) - repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false) + repo2, _ := NewRemoteRepo("tyndex", "http://mirror.yandex.ru/debian/", "wheezy", []string{"main"}, []string{}, false, false, false) s.collection.Add(repo2) r1, _ := s.collection.ByUUID(repo1.UUID) diff --git a/deb/snapshot_test.go b/deb/snapshot_test.go index 85bf14ca..8d1405a5 100644 --- a/deb/snapshot_test.go +++ b/deb/snapshot_test.go @@ -17,7 +17,7 @@ var _ = Suite(&SnapshotSuite{}) func (s *SnapshotSuite) SetUpTest(c *C) { s.SetUpPackages() - s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + s.repo, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.repo.packageRefs = s.reflist } @@ -116,11 +116,11 @@ func (s *SnapshotCollectionSuite) SetUpTest(c *C) { s.collection = NewSnapshotCollection(s.db) s.SetUpPackages() - s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false) + s.repo1, _ = NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false) s.repo1.packageRefs = s.reflist s.snapshot1, _ = NewSnapshotFromRepository("snap1", s.repo1) - s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) + s.repo2, _ = NewRemoteRepo("android", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) s.repo2.packageRefs = s.reflist s.snapshot2, _ = NewSnapshotFromRepository("snap2", s.repo2) @@ -200,7 +200,7 @@ func (s *SnapshotCollectionSuite) TestFindByRemoteRepoSource(c *C) { c.Check(s.collection.ByRemoteRepoSource(s.repo1), DeepEquals, []*Snapshot{s.snapshot1}) c.Check(s.collection.ByRemoteRepoSource(s.repo2), DeepEquals, []*Snapshot{s.snapshot2}) - repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false) + repo3, _ := NewRemoteRepo("other", "http://mirror.yandex.ru/debian/", "lenny", []string{"main"}, []string{}, false, false, false) c.Check(s.collection.ByRemoteRepoSource(repo3), DeepEquals, []*Snapshot(nil)) } diff --git a/files/public.go b/files/public.go index 5739c8a9..fc87b410 100644 --- a/files/public.go +++ b/files/public.go @@ -123,10 +123,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress // sourcePath is a relative path to package file in package pool // // LinkFromPool returns relative path for the published file to be included in package index -func (storage *PublishedStorage) LinkFromPool(publishedDirectory, baseName string, sourcePool aptly.PackagePool, +func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { - poolPath := filepath.Join(storage.rootPath, publishedDirectory) + baseName := filepath.Base(fileName) + poolPath := filepath.Join(storage.rootPath, publishedDirectory, filepath.Dir(fileName)) err := os.MkdirAll(poolPath, 0777) if err != nil { diff --git a/http/compression.go b/http/compression.go index 9ee8f865..eaec2d5f 100644 --- a/http/compression.go +++ b/http/compression.go @@ -4,7 +4,6 @@ import ( "compress/bzip2" "compress/gzip" "context" - "fmt" "io" "net/url" "os" @@ -90,7 +89,7 @@ func DownloadTryCompression(ctx context.Context, downloader aptly.Downloader, ba } if err == nil { - err = fmt.Errorf("no candidates for %s found", baseURL.ResolveReference(&url.URL{Path: path})) + return nil, nil, &NoCandidateFoundError{URL: baseURL.ResolveReference(&url.URL{Path: path})} } return nil, nil, err } diff --git a/http/download.go b/http/download.go index 5f5e802c..73fb189b 100644 --- a/http/download.go +++ b/http/download.go @@ -64,6 +64,29 @@ func (downloader *downloaderImpl) GetProgress() aptly.Progress { return downloader.progress } +// GetLength of given url +func (downloader *downloaderImpl) GetLength(ctx context.Context, url string) (int64, error) { + req, err := downloader.newRequest(ctx, "HEAD", url) + if err != nil { + return -1, err + } + + resp, err := downloader.client.Do(req) + if err != nil { + return -1, errors.Wrap(err, url) + } + + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return -1, &Error{Code: resp.StatusCode, URL: url} + } + + if resp.ContentLength < 0 { + return -1, fmt.Errorf("Could not determine length of %s", url) + } + + return resp.ContentLength, nil +} + // Download starts new download task func (downloader *downloaderImpl) Download(ctx context.Context, url string, destination string) error { return downloader.DownloadWithChecksum(ctx, url, destination, nil, false, 1) @@ -81,15 +104,10 @@ func retryableError(err error) bool { return false } -// DownloadWithChecksum starts new download task with checksum verification -func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string, - expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { - - downloader.progress.Printf("Downloading %s...\n", url) - - req, err := http.NewRequest("GET", url, nil) +func (downloader *downloaderImpl) newRequest(ctx context.Context, method, url string) (*http.Request, error) { + req, err := http.NewRequest(method, url, nil) if err != nil { - return errors.Wrap(err, url) + return nil, errors.Wrap(err, url) } req.Close = true req = req.WithContext(ctx) @@ -100,6 +118,18 @@ func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url req.URL.RawQuery = "" } + return req, nil +} + +// DownloadWithChecksum starts new download task with checksum verification +func (downloader *downloaderImpl) DownloadWithChecksum(ctx context.Context, url string, destination string, + expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { + + if downloader.progress != nil { + downloader.progress.Printf("Downloading %s...\n", url) + } + req, err := downloader.newRequest(ctx, "GET", url) + var temppath string for maxTries > 0 { temppath, err = downloader.download(req, url, destination, expected, ignoreMismatch) diff --git a/http/fake.go b/http/fake.go index d53c1700..4026791a 100644 --- a/http/fake.go +++ b/http/fake.go @@ -60,8 +60,17 @@ func (f *FakeDownloader) Empty() bool { return len(f.expected) == 0 } -// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification -func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { +// GetLength returns content length of given url +func (f *FakeDownloader) GetLength(ctx context.Context, url string) (int64, error) { + expectation, err := f.getExpectedRequest(url) + if err != nil { + return -1, err + } + + return int64(len(expectation.Response)), nil +} + +func (f *FakeDownloader) getExpectedRequest(url string) (*expectedRequest, error) { var expectation expectedRequest if len(f.expected) > 0 && f.expected[0].URL == url { expectation, f.expected = f.expected[0], f.expected[1:] @@ -69,14 +78,24 @@ func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, f expectation = f.anyExpected[url] delete(f.anyExpected, url) } else { - return fmt.Errorf("unexpected request for %s", url) + return nil, fmt.Errorf("unexpected request for %s", url) } if expectation.Err != nil { - return expectation.Err + return nil, expectation.Err } - err := os.MkdirAll(filepath.Dir(filename), 0755) + return &expectation, nil +} + +// DownloadWithChecksum performs fake download by matching against first expectation in the queue or any expectation, with cheksum verification +func (f *FakeDownloader) DownloadWithChecksum(ctx context.Context, url string, filename string, expected *utils.ChecksumInfo, ignoreMismatch bool, maxTries int) error { + expectation, err := f.getExpectedRequest(url) + if err != nil { + return err + } + + err = os.MkdirAll(filepath.Dir(filename), 0755) if err != nil { return err } diff --git a/http/http.go b/http/http.go index df6ccc50..480a7830 100644 --- a/http/http.go +++ b/http/http.go @@ -3,6 +3,7 @@ package http import ( "fmt" + "net/url" ) // Error is download error connected to HTTP code @@ -15,3 +16,13 @@ type Error struct { func (e *Error) Error() string { return fmt.Sprintf("HTTP code %d while fetching %s", e.Code, e.URL) } + +// NoCandidateFoundError indicates that now candidate of given url could be found +type NoCandidateFoundError struct { + URL *url.URL +} + +// Error message +func (e *NoCandidateFoundError) Error() string { + return fmt.Sprintf("no candidates for %s found", e.URL) +} diff --git a/man/aptly.1 b/man/aptly.1 index ec5d991e..a534b42d 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -482,6 +482,10 @@ disable verification of Release file signatures gpg keyring to use when verifying Release file (could be specified multiple times) . .TP +\-\fBwith\-installer\fR +download additional not packaged installer files +. +.TP \-\fBwith\-sources\fR download source packages in addition to binary packages . diff --git a/system/files/ubuntu-archive-keyring.gpg b/system/files/ubuntu-archive-keyring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..b18548d05852ea37cd60e1d6a015c36eafc9a271 GIT binary patch literal 12335 zcma)?V|b;@+O217+qP}ncG9uUj_q`8JL%X~$7aWNI(E_>p3d59eS7V-&v&l#d(L^k zRnI$SJ!4c=I#2Z4)br)O%B zY3{Bm!~Fn)=O4;r8SQ=guxp9zG*f;M_;434$*opL#zC<`Om}JgJNZZVzvAN}I2Y+6 zHqng;jlV&~4%SgARVWgH`qY}(nu^4zzvVmwZ6k^O(4Q|+d4pTboOUuq>|r-*wfxbA8D>3EyJ zef3jLnu!yqMJ%0D`eq~uaHM`HMksmg>ARO%QjevSjov0!JZQT`Y|WRV>ZzEbYt*WlTK? zdCgoLYz>`VOq~Q>|M)(=vAr#S_$N>lBwzqO81Tn?2F3vahk=5G009O;0|o&>2L=Mh z2Li_M2S7t13Z~lCi1>gtkxl|~ReBL6A0IXFE0~89_O%F;uy8S0FVe^Tu>AsU;y~S)sXPx#XyhNNdFUvR%{xh~@^1xD5klX=- zg!$0g;Z@E4b#2XT@y1B{Gxs*4ueR;#%CKgdDWg_OcHyFU1!_W zh|1T6t53EgZLHmfLLA|DP|aOfaLfni2Ry0@{${crQy&HRrp1{%lIifc$&!s^gl(ya z#%-Jv>(bfabL(h;YS?ZSbnX)6+$wdiNia>9CUvOqQEqH2bp8th_do;VO`w)`t0kZG zhrCy3xJy*SUQXj8=y{LZr&|hgt-=ncowqIsT~(8rVz9)efRK`~8kPe}W}g^8j;j)1 zFNl*1dpuIhOu1L~Zb;O5(fG2WPkNz{58tZAB@;@;-J0%^?rUG4DO^0mXrQMtT}QOc zewgxUA>i{0T(TeB%LWCxIrZ=`ze=Ae?D)H1Qw|u1o@n(^1aQ-+IL_2v0 zH6R-=l&s^hD6D@O6iRFdXDBIxI+OB0Sz-`w%x?iUEL)H~;DEfCExjXPO}iIO z!jUj9LVM;DYd0eC<*cKbAIYut6S#(^T^J%nbk!=)$_ATMe7W^78^E_UZQElWN-g&5n15b?iQ z!XGRcE=Ys%QZp>{L_ii4DkVt0pfk})AmY9Z@Z2hBTvQGqhl2Cg&z~1Z-B89)^EHBp z4jgDB|G!uUA1wOHl(=HWqu>H!K;G&UY16PvI^lk2X?QZ0I_G)`{23tE^>h&xua~YZ ziju&wzV2>rD3>+)A1w1#?+;e;E<}~DjBcu`Hz0@U%1Hq82DixBj-r1)DWN8sBy9nZ z!!bfuz8xme;-gRaX*Y+hP%PU9`X8)3AgT|RRzTZan9S)2H8ddCk1Ajq)CRl_A6;Td zy`A}l_^507x9a4@Ll_HhM)qOsIrdRXw7&0Jd;f#gh==#V@)2L;J>P3GlU#TT(v#3k%ZU;@1D5+PnLM~ zL-q&jVS@!2@kw^pI|q;}jFm0R>WW#L`Cz*0%wMXz=7|0Skd4T(zdyPU6!H+Y^cC@; zB5y#>ncz>>M~nIW-xkge>Hc6bTC0f;#q&PE!~AX~#w$(;UY#DUyQ*@o({PO`HQL3djpxj?T&U^s4qZB920f z>dqhb-*^DzHP{+rXU>#IMKu&Q+_TSYlayBp{K1m?W6gmA0~X74_y!CC_!t^QG+A^# zVbu_E03cVIq5eEREtzfPB`(3IBQ{Q74yG267o03r!q#^H59)4yGi3)$Nb$g={ek=F za)|%xauTdV$saJ7=U_hp3I-k2N<=_yT`Jt6pP+Y|kKuH6nlGTXYY!_k`L6mJp;wfT zPlIGP!xO2AM8X^vM?U*yY*6*i-mK8WKtz66@GW&H0Hg+GBN@w)Xm1yhsoF%)e!U~Kmz(kRS^h8oqscn z#PumEvi~4AkcxO7J66+*K)&?>=b7#X>wzB8puixg+3kK8R2kK4g!M8TzQUj zSBv@~JmeFhOQ(=&Oz{`O$PG`}D&zofQ^LWC_#6ssrqQ|wxQ<_>_i@l&ax!n&`WH^s?}`c2w-x9ROzx6 zm0@@M(_A3KRz_K-^M;reN^^JEE)}0`h7<*C?~6t7pmViHf=K8&m(B!#pk5R}msYpu zY0p&@OgDRG!F<;i3d3{XILa|1ZbO;Y*seQe-G3Ltmzd6Q4m(PD+(Ruw2O-bG^*O8o_Ijok z`*v~kd6pK~CCUY0&KSs^jR&IbSDsCW!-!N<^?X?qUba&(yp`a->MjeRmX-($Jw0p4@#p0hvvQ~N0x*) zSF4yPr*1v%F5ad3!<4JyLhTg5+bQGSUeJ71{v;iU|tNR7$Vw1 zQ!#U%%|n);pSe;CvhuD1^R8;)y^- zd>Cgk{LXk@{R>N8y&i*4$=2Y+QHVfz5s>#2%V4z#19YA3zF%coMj4w9qJ;vbAFeJd zsMMmNq2e+8rEN8S2G&VEQLVwWxjCmq8VOY%bm+PKZ6B!QK;MV1q*XxwnOsB^P;Q$4~ze`v=(X;S|+^N6$u9?2Oux^O~-vQR;dvd{~cJ){4qNu80{StkW{MI zvbm^$^n;*4x_mXoZj|&%OnkeNBq5_|f-R+4eSW2h(qv-XrXk8sL<|x!6MY9BdJhDG zUx!U5GWm$@9&_oM=hG;o=`Q0hcHb|kQU<8XIx?+2{5WNoFT}xm56KcdaZZad4!{g4 z*$fY1?=uEcDapcXg+;xdUMlJILm=-)-zr`(ehpTK>3d@`Za1c9Eqr!u7ANjDeEW>R zL}&19g64!nLqWUn}kX9I!Mon%#g`WfD;lXe$e>A@H$5jZ!a57!x)hwgf3 zkYjLzkhPn?HHvwdf;SQ2YOW^N>@U8@gTbMz~FiCo6MUP{1(&?h~dt8Q|$gd~*SFLDv zB5ON~#-28#4FAdc?Zp3ZH-9wg>C8C&jivC zfE(S5pgT-q+CXmu$4mUsP{ty^E290XdA-jtoVL0sdq1wR~$L z#iF&|rxt(U*8j3zBK9WuABCG;Q3i!N&R>{^0|zt?((7B7U^kS9QQC4raPP$n481#r z5*YWs*0GCYp?MgSMT)uV`YqW5yd8D5!K4k2AQK*lKp1)*`kK%J*4a8YdiQoGxsQ=A zT_$_&+gV`x$)E{O1p7Yk{^%${ld=C?8(#|e22(g?#6qzq7Rd>wG3(d00q+s&X`aoh zGLtpUFLC+E2x;n<%>un-@12?HqS!|4cJk@;fOQHi@W`~{Tv3seF*RogYPS+A2Ovkc zQG?sndNGF56mHmSZ~yQy!w8`NahIZdUmrfEXlYLDOxqt%h|Lc1Ch!S+QYnk2MD4Jv65CxX=-va#WID_Xwt#T?(`<^&OXx|1ec>$ zlTe*09($q&;zU*N^t&%FZAhKs>Sf6nW}X;KXR%J@<>=4MSq(gBF4aY~#TAAzJ19>Q z!3r=5v=1lm(qft0jg!1nijUT1uw=+F&&_02@J_fB;i(y|GvJoSZxyTM>b-Y_T&r(~ z1-93&W;r0>#b57Eyoi1!Oc_`=paol5j}7%!)3OdI^rm*N=JJ9TQ=ICF2paF6*d})z zQ2TaZlk+cupTAzYd=EY|B=>|&_euPP8jR>3ykr)3;kvOQm=XP;< z>^vma%A%jiO=q9HfEg(2^q-ZB{RqB$k^tB1td;y#BF2AZ z6w$qMqQ94D2?65W`te>NhI8GW2Uf8I-KB>MqFLW>KhK|OY8LusR!{$N*AQj8S-qt* zRz4Ye9(rKh!~O*1`O8&YPf6d&RCx9sLfo*cT=o`ER&$i_;_v`-?lB&GFFFVhIK1~E z--|azxzCrLZ|lsL<>dMZvs%vDF_CVPKNy%Y%6#-xKkw|Y{SaXY5H9SwSrNd{-MB2s zBqRu@abvbjY&|33JaQ2^J5X+@xn@371B>ok@us@#w1z+^n#YSUKcCiC0+Moapmg?x zYgYGRv!5ZQh)pF_jPN?^2SMvGf(WT#Z{dEi<6x>`#zo-!*&P>97Mf-wG4~vax1Y&4hqF8 z4s{QNE-EgQKGtDi$A&o%xf1T?uijvfadlpemPI+wQ|}<YToUD z%Nh6-Xo$x8`nF!C0EvQ%1nt{L}l8%Y+G3Od{$k$iM`C znR?Rnc|{xH5>!V=51a;%;2$OWkEEOPQKG++?q&(Xn}E%W$y_`o-?F+xX-DFaLzq(9 zwGG_H#5>QVoVU!tl*3z}mravwG=`n;fg%DdGj>ykfFIjeJ-lMxz?%zN7Z~tC)cuM+ zfqgHbm9uk)=8JF%UmK96&5kdusd|D7?6kF4U5ptp%1&D}oKNnMP5rKD87&jm>sKm%Mey$7!eKhw#gAW266W_e zbGOd4Cc6AIowo#Ub=H4*!#A7<38(_r{1yD89dfokrJFi3`?CXCEOOTu9^u-rYc`?Z z?X7i#U*YR0tpi)WI^HK=$dlsUg)uA5ZnVvcwu0@dRaF6{X#+xb!lH$iX#1bbpcOEz zdNhNlayAFdMR;a*Stv(E+`Ow$5Fkldj@4ybhNuA6x-$B~dD2WhEjg*MRL{WQzzabG zg-re2E&_IRXRPsK3wkD{Qw(QRJRVoDYjQKFosV!vfn~iGv=0JBJ#TM4Zq&pjqM2=! znt%y1pIT;9MJK?ouTWUaf+j0VgTX_hs1}u1$<-iZh}p*oY4HZMVKZWm17DlRcXRB^ zoyaIR@6be;TuL(Khhw$o01yTXJbc2#C08F3N&lU66{-TytTb8({3Kq;r0NRGy+`y zlfR@syz29>h}N%rBaZ-`jfI)fRC5SFnivjWzgw>*D(^egCnAvsueisNr}6_#bZ|}G zzE>dd=<}ypxU|BPE#@ylNFK5Ep=xDs11KXTzOK9CWeqQwT!!Gk?4X8h>I(3zjb*lh ziRR`je)`6!mL0WT-2{_1j<^k~VCC7 zEd#oEsmni!6_X^NzE8z#YYDB>PznGM@O+CRvyDo2+04B*srZ8Ot6VSmbAdK#WyOsL zr^MD=eHa+r7EQ~I`hi?!yv{N;l?Cz*4C+P6i_KD1VYG-v$p-G(Ni=xS59JUs5H#fC z9YzrNy2>eg(X%Kfiu~7z?`^o9+CSs1QiP3SN-Y`FjsQqN0H8+U|F1JDN@hkTW~%?? zs44=O^N(XBVldzjY5hC?fTKYILqWrUq635c?YxRd;jr>sJpr?xFCrj*-laOnU?Re) zPGO5UwMiV2IVDVVdMhO|y>Z7d#T-U;1c>$I&b^~oMOfFI+!A+u#fh{ZbTIWQO4v|y z)|2CoMoOX6aTY6|r=>tbO)zK5SqFA>iZ8?=Sy|7dA>+rd-~-czPwBjEjJ(};@jO;@ zYhDBAa#X@V(0V?X!a!N)b)%NQe?JsI|6LE8fe+-Gzi z+gy;*NesGpa{2ddJ;dqP)*152iXbt*+-7EZnWQ?;GMTIBw9E;w6(e~c*VvCjojx%= z@~Y5bBe)+0E#9arh!=ksLuSgelrF3wx2S&sn8$9UV;(X}X6o&!VlqH@7_?{}s9p8+ zha;4Pr#tY!AcVtdwdP6gBcNeOPvY6lsP}fpBgDeGBe{*{m$JqOQNhjsgg%y4b|`+K zFO-I!D{B?`RwCYk24}a8jYZ9Q435qw5q#{e1xs{sqBFQv9vb_&DvIE(;e^+Kh9VL9 zU8?3Rj?|asEw)+t*LM)hT_vUiG{S&Xoads z-H)!t#TK^W6RHT)^c~?{-_+D{laojkZ4LTIw0?wWMfFIx7)I`Z$uA8sbUARynkGis zgdu)`(6~nYm8IZVowp3LboQZ_^ow3`m=<`j|`h_ltp%E!LT&F})O-Pt)VIku=t8RcX5RJhjVRR@zfWwa}HmVLLM zx+`L8cgA0;AdclZfy?F!A#gOlb{BD{q{I*JxhN5262zu3A`)KaRC)G1hCTRvgO=on z)&sdR8BnEv^og*;NUyj=*MnQ?yWe}cHz~t@oS1<%DAtt>P@F>~`fL(;?RiTjg22li zgi)pn5^ldy%$i`dtC^OGW>}XbFGHUc+fO&}`C+aHoe5>6h0i{uDc%Aqsr*Y?2Sz7h;d|y@9<_yH@w@N8UXWf-)1dD2jTmmZ$A&UHV zT$Xfb{WN2N30YKWvrTib#anAfkNo)DJ1(XxNY6?k%ErVW9WRng$0SGhUB)7&`>G0X zgSTW+)mq-xNz%AHM>OtBlfSp+niYtdl4vJFGRGcO%#Ep$X%y4;K2IyMw8v%5>Z#Pv zZUi)XvHmK`?EZ(bQK*VA`n$0)w?YIIy(MkV;)Xkv3M961_mUBX7$INOB)y`;xuykX ziY)7?enY-@?()$e-afL3;_uELS|q+?$$F-?=8VlJq7UQ;uEbcF;95suiSeWvC8@4& zrzmR3p57ay_qTvux6P1Mq-Yq?fCfzlOLa#;LF^@PxcgQ838=JA898+HRldRbmqfNOMtZ}LZoGkPD{jIR3f#r(b zOi`r1;_LRJTVv4>z0~_Bp*}FUV)(Vd`=;QdZiswg50-VCsQ8nDavOU44ivqJd?gxZ zms@r=$4ov|a5k^D7MhFBTHK#Wr@8(h#Eku08JO=;Mn7AC6V8G<*m#l5VQuVZzUnOy zn9Y+&E9@{y#_ul1^>;*k59E65&RLtWFlzyCn{AlMNn_5?Aglc{C#u(ue7k!Eo?OC# zu*!CK6tZ$;P)wpJRpaP9iG7Awjv|N z_6tiVp8OSZLPomrG)ymf3WPe2zBi9#ki;tfi*s;uKZf_w*cK6ZQ0qc`shyOk$ywxd ze?BVTh~lEM@UO$qW>6q;q*Wx-sO!c=>kQm*7P;YHL9`U7QAMfO#LAWIY?lTQ#N7F? zQsAFi4bn&XEc=W1aoS*#mPFTj=Zx-S)j9uG+Z+4@e(Eip!tvbax=Pw6p>! zDw{Kbr*G_&ALvI4FVAdcloR5_7_^vrlO7 zpucKCissd|L?U_94$_ROXgf>P*4L$7z=}aTzgc)`OqO(}4R*ST<#9l^i{0p#U~l6~ z7*J`BwM#*kVXstjgX~{?TN*>LcmGm}9Og=&N1Xb(k!~$ zNsQ_5l6o(5G_1j@G8^AZ932VR2Omy&?5|twyH`My@bm>hN}H_`fXBwU50KR0LuHVj z3E8*=HCAP#=-uXK4}uS1-0ZkmGUr``Y3y?0&@PB9c2L<-7$LHLpZO7ao0R!T7`NXHnCis{ zm;xu3=F5vZlGA)2Cci`l4Qvsi1s4-RTn4SZ(8%Zhl1~8if<-tV>=l%iIa$+4K(nC7raSFWTE5ChbRTuw#v`WmQnV-Xt2l zq(z0aJ)L72mCTXLO8}V-KiB3c?GSwB#=zM*LTuY&mNO|Ucu5tr_n{wRGIx~9X*ybt z&4a)N;vA^5$b->c#O=?MTOAzp)4PrS9a$BaUdIXb#U9?oiev`9mT3EzFQ>KH((a^X z=}k5BE8@}$BsbM}%b~fqG(HcYQr8jaoThgqAc2gjaIC43nxM?%&P9Xdb8GPLT=#+n zb#9%^3u)SIvKwJT=+h<(<|T|h^q0%*ur-7mD*FhVhzDgSZP(zL-Oz6mF&X2$j?h5* z3ks-X%8Q1=3g<~K#&^oMWB?Aj*?e;l7@hlnd0(Obza9U-%2hz!TnUK>w zp_qp}Kk_A0g-s>&^(a|%Iex`DNR8$wMURB=IE1SQkas7VH`;S?3KTc8V!vFoYa!=% zl%tZnta)Umn(=Upv+<=dg*WFS6ge(mZab+WG=vi1$Kp^Sau(|Ge77994&ZmAmGlQ? zJZ-!Vtu}I5x6#hbrmm4%xY^EJ-tR{aEBxlWnpLpa8i=#5?v%a$@HzZ+7%br~u5?^a5PI_z&)Aq84-Dl)fe0EYi z91%r~{9Lr&1ga<}i(KEq<=I&qA zqp}vhDN9{~#WtcSh125E@Y$+Cfos#M!0CO)pAobX==78@il2y)unnO9;>y#b9t!G< zk*+aHWGyz6#>XWGbS=A>7uH*R$77qMM25R&-(klM*cih8!ss3V;PQ#ikJ&Xzo@MlH zP1d_B5?t3TQP*z&iZC(R&)$M(W7>u^T=?1iDesyRT%wvVBdPX2fDU-B%z`^Rx^6uj z)d<;YlMNwLa090Nwawq0VYRwF3r#Swi?^YEsb?*`5Ksad1@t|Lao_~vnLFa-M$Vzz ziT3yuBuZ%ux;bD$;C{{11!B;yA9C;Rq>hB|>4-AEmc(JDmWCQHeaHJ;R@(Ui%2%n& z_Js9zu#y3t7M`7m8$105hll@LqiJ0gwx=H0w<7aijxQ&~Him~|+*T@jL6F}=rnT^$ zpVdPd4S!+%DpeNclVbt64qC!mrof@r{9@UNiZ`^k^Tl09oJBsF|jzwJ;4FZr#20Y_mRcNigH)#ZzAK68EW`ZsOgtyC%K0L z6BX<=y;?}KntDS0fI;)?vMmHvHhCCSbjMV-4lw^bR0k&AT+lQ28R5 zIaUG_iV8Ua+1=Gz!95jAyE2~onxORYLQ&MF*4^ecO85st`p1LoyT_AWZ)N&=>A^md zX7(GsIKv^k#$^J6iJ~I~qJ^a#0SE*+vZh^mCbROb`d;f0ga)B}$6)9ljaH~trm6`E z7qc|GQ>%LlTj@BD+CFLjL^+gWN+!5!^~M9`eaDm&bK3s0FuT{nZ_>2~oib>-L9mg&GZSR2?04DB-RJ&-x7iT zBR^MTbA3oeL+e%%s9f6QCJf=-ORKaY!GI10<3M!@Al99f@p%qmI#&l)%n{$|ORUhH zIB})=&aU204%D{l*-=Jslv{q`E)9qSeR4@``Ci#@cJn=-F~4~&^O=r={)%pkXj~Lz z;Qp(y9-8byF6af`>W^-AeBTtKI$2tNC5XoFfCj5gtzIEoX6ae!%K$=92a(H$?OK2t zmfo!Eh4chlY0Y(vBU{>(+d(*w;ZpBnS6>K%eNps#LPO$E?9GNeKb7OG|9sh zA4hy?aW6CSg{*;#Z@r&=RbkTXKgR-2We)|tIqE7?cHZ1H@Gab~B`C=S*MyP%>0XoR zf9N|(_hFTQ+eMpMw!dg6&pS!nEJ6a~SHQniX*dlOdC%=TVZw0^NXJjw@u)k<099QW zaaKmQv}<7VQZG*G#ab8Mv)wd8KSI17DqY@6xG<)0BbUw#`Q>;Ub6^+*>^-JzOV3S= zw^Sw2qZ%vVTY%moTVw&w!Ust0$<9W+&@o1MqM} UnOq+SNSh5L9s@*gi4TeVAAw^TU;qFB literal 0 HcmV?d00001 diff --git a/system/lib.py b/system/lib.py index d2cd4f12..7db35471 100644 --- a/system/lib.py +++ b/system/lib.py @@ -84,6 +84,7 @@ class BaseTest(object): fixtureDBDir = os.path.join(os.environ["HOME"], "aptly-fixture-db") fixturePoolDir = os.path.join(os.environ["HOME"], "aptly-fixture-pool") fixtureGpgKeys = ["debian-archive-keyring.gpg", + "ubuntu-archive-keyring.gpg", "launchpad.key", "flat.key", "pagerduty.key", diff --git a/system/t03_help/MirrorCreateHelpTest_gold b/system/t03_help/MirrorCreateHelpTest_gold index 8830ee1e..10e191c9 100644 --- a/system/t03_help/MirrorCreateHelpTest_gold +++ b/system/t03_help/MirrorCreateHelpTest_gold @@ -28,6 +28,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) diff --git a/system/t03_help/MirrorCreateTest_gold b/system/t03_help/MirrorCreateTest_gold index ced80808..7b138a51 100644 --- a/system/t03_help/MirrorCreateTest_gold +++ b/system/t03_help/MirrorCreateTest_gold @@ -19,6 +19,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) ERROR: unable to parse command diff --git a/system/t03_help/WrongFlagTest_gold b/system/t03_help/WrongFlagTest_gold index 8824d9ae..811e4d64 100644 --- a/system/t03_help/WrongFlagTest_gold +++ b/system/t03_help/WrongFlagTest_gold @@ -20,6 +20,7 @@ Options: -gpg-provider="": PGP implementation ("gpg" for external gpg or "internal" for Go internal implementation) -ignore-signatures: disable verification of Release file signatures -keyring=: gpg keyring to use when verifying Release file (could be specified multiple times) + -with-installer: download additional not packaged installer files -with-sources: download source packages in addition to binary packages -with-udebs: download .udeb packages (Debian installer support) ERROR: unable to parse flags diff --git a/system/t04_mirror/UpdateMirror23Test_gold b/system/t04_mirror/UpdateMirror23Test_gold new file mode 100644 index 00000000..845f82da --- /dev/null +++ b/system/t04_mirror/UpdateMirror23Test_gold @@ -0,0 +1,33 @@ + + +Applying filter... +Building download queue... +Download queue: 12 items (25.14 MiB) +Downloading & parsing package files... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/InRelease... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/Release.gpg... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/binary-s390x/Packages.bz2... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MANIFEST... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MANIFEST.udebs... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/MD5SUMS... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/debian.exec... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/initrd.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/kernel.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/generic/parmfile.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/initrd.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/kernel.debian-nolabel... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/kernel.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/tape/parmfile.debian... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/main/installer-s390x/current/images/udeb.list... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/non-free/binary-s390x/Packages.bz2... +Downloading http://mirror.yandex.ru/debian/dists/wheezy/non-free/installer-s390x/current/images/SHA256SUMS... +Mirror `wheezy` has been successfully updated. +Packages filtered: 35003 -> 1. +gpgv: Good signature from "Debian Archive Automatic Signing Key (7.0/wheezy) " +gpgv: Good signature from "Debian Archive Automatic Signing Key (8/jessie) " +gpgv: Good signature from "Wheezy Stable Release Key " +gpgv: RSA key ID 2B90D010 +gpgv: RSA key ID 46925553 +gpgv: RSA key ID 65FFB764 \ No newline at end of file diff --git a/system/t04_mirror/UpdateMirror24Test_gold b/system/t04_mirror/UpdateMirror24Test_gold new file mode 100644 index 00000000..ff280c29 --- /dev/null +++ b/system/t04_mirror/UpdateMirror24Test_gold @@ -0,0 +1,61 @@ + + +Applying filter... +Building download queue... +Download queue: 37 items (166.14 MiB) +Downloading & parsing package files... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/InRelease... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/Release... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/Release.gpg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/binary-amd64/Packages.bz2... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/MANIFEST... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/MANIFEST.udebs... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/SHA256SUMS... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/SHA256SUMS.gpg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/cdrom/debian-cd_info.tar.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/cdrom/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/cdrom/vmlinuz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/cdrom/xen/xm-debian.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/hd-media/boot.img.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/hd-media/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/hd-media/vmlinuz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/boot.img.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/mini.iso... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/netboot.tar.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/adtxt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/exithelp.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f1.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f10.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f2.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f3.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f4.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f5.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f6.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f7.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f8.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/f9.txt... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/menu.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/prompt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/rqtxt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/splash.png... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/stdmenu.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/syslinux.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/txt.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/boot-screens/vesamenu.c32... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/initrd.gz... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/linux... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/ubuntu-installer/amd64/pxelinux.0... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/netboot/xen/xm-debian.cfg... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/main/installer-amd64/current/images/udeb.list... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/restricted/binary-amd64/Packages.bz2... +Downloading http://mirror.yandex.ru/ubuntu/dists/trusty/restricted/installer-amd64/current/images/SHA256SUMS... +Mirror `trusty` has been successfully updated. +Packages filtered: 8616 -> 1. +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key (2012) " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key (2012) " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key " +gpgv: Good signature from "Ubuntu Archive Automatic Signing Key " +gpgv: DSA key ID 437D05B5 +gpgv: RSA key ID C0B21F32 +gpgv: DSA key ID 437D05B5 +gpgv: RSA key ID C0B21F32 \ No newline at end of file diff --git a/system/t04_mirror/update.py b/system/t04_mirror/update.py index 5f63ec35..bba29284 100644 --- a/system/t04_mirror/update.py +++ b/system/t04_mirror/update.py @@ -376,3 +376,35 @@ class UpdateMirror22Test(BaseTest): def outputMatchPrepare(_, s): return re.sub(r'Signature made .* using|Packages filtered: .* -> 0.', '', s) + + +class UpdateMirror23Test(BaseTest): + """ + update mirrors: update with installer + """ + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=s390x mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer wheezy http://mirror.yandex.ru/debian/ wheezy main non-free", + ] + runCmd = "aptly mirror update -keyring=aptlytest.gpg wheezy" + outputMatchPrepare = filterOutSignature + + def output_processor(self, output): + return "\n".join(sorted(output.split("\n"))) + + +class UpdateMirror24Test(BaseTest): + """ + update mirrors: update with installer with separate gpg file + """ + longTest = False + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=amd64 mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer trusty http://mirror.yandex.ru/ubuntu/ trusty main restricted", + ] + runCmd = "aptly mirror update -keyring=aptlytest.gpg trusty" + outputMatchPrepare = filterOutSignature + + def output_processor(self, output): + return "\n".join(sorted(output.split("\n"))) diff --git a/system/t06_publish/PublishSnapshot38Test_gold b/system/t06_publish/PublishSnapshot38Test_gold new file mode 100644 index 00000000..0365afcb --- /dev/null +++ b/system/t06_publish/PublishSnapshot38Test_gold @@ -0,0 +1,13 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... +Signing file 'Release' with gpg, please enter your passphrase when prompted: +Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: + +Snapshot wheezy has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ wheezy main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/PublishSnapshot38Test_installer_s390x b/system/t06_publish/PublishSnapshot38Test_installer_s390x new file mode 100644 index 00000000..dfbcaf29 --- /dev/null +++ b/system/t06_publish/PublishSnapshot38Test_installer_s390x @@ -0,0 +1,12 @@ +a29b0c17f98afb5ebc5a65bd03411e430dd372223d931b2c2441604fd6942472 ./MD5SUMS +a5a8e9029fc31df4a892644524a28c7d31f12a012d72cf1ac5672178b982ecac ./udeb.list +d592bca155b709066bcd48070358e6a2dd4f401c09f594b37b15caaf9fc45d80 ./tape/parmfile.debian +5e4d45f22a87e36d21685043f85106a6380b3ae7fbdccff70097ccbf6757a287 ./tape/kernel.debian-nolabel +57afafa42e61248d96e9d26efb2db075a17f484c51445c6420a32220b05c1fb2 ./tape/initrd.debian +0ca15998b2156af0a2ecffc771f9a418b4918956049fac4250c01d163b143006 ./tape/kernel.debian +1a907f670fc825f70827608ec32bb8a194971d0e1126d1caf33de87fc27d0d08 ./MANIFEST.udebs +62ba2aaafac85c163c3cbf25ee52724a59b12d805382a0122080b6a86e229b0a ./MANIFEST +d592bca155b709066bcd48070358e6a2dd4f401c09f594b37b15caaf9fc45d80 ./generic/parmfile.debian +48d2cbebbc8582f546232c1acf68b1b73125731441fda528f8d71795a60e14a4 ./generic/debian.exec +5194c4cfc1d527ce6aa45ced5a7f5102a3c85851ac8409c32b53c1004d0717da ./generic/kernel.debian +57afafa42e61248d96e9d26efb2db075a17f484c51445c6420a32220b05c1fb2 ./generic/initrd.debian diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 9ed1f6f3..fe8f2461 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -1021,3 +1021,25 @@ class PublishSnapshot37Test(BaseTest): ] runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec wheezy" gold_processor = BaseTest.expand_environ + + +class PublishSnapshot38Test(BaseTest): + """ + publish snapshot: mirror with installer + """ + fixtureGpg = True + fixtureCmds = [ + "aptly -architectures=s390x mirror create -keyring=aptlytest.gpg -filter='installer' -with-installer wheezy http://mirror.yandex.ru/debian/ wheezy main", + "aptly mirror update -keyring=aptlytest.gpg wheezy", + "aptly snapshot create wheezy from mirror wheezy", + ] + runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec wheezy" + gold_processor = BaseTest.expand_environ + + def check(self): + super(PublishSnapshot38Test, self).check() + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS') + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/generic/debian.exec') + self.check_exists('public/dists/wheezy/main/installer-s390x/current/images/MANIFEST') + + self.check_file_contents('public/dists/wheezy/main/installer-s390x/current/images/SHA256SUMS', "installer_s390x", match_prepare=sorted_processor)