vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-07-15 11:57:59 +00:00
Code Issues Projects Releases Wiki Activity

New upstream version 1.1.1

Browse Source
This commit is contained in:
Sébastien Delafond
2017-11-02 08:56:43 +01:00
parent c926f2bf05
commit ee9fb8dfec
356 changed files with 656439 additions and 643115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deb/remote.go
+103 -74
View File
@@ -7,7 +7,6 @@ import (
"net/url"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
@@ -18,6 +17,7 @@ import (
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/aptly/http"
"github.com/smira/aptly/pgp"
"github.com/smira/aptly/utils"
"github.com/smira/go-uuid/uuid"
"github.com/ugorji/go/codec"
@@ -45,10 +45,6 @@ type RemoteRepo struct {
Components []string
// List of architectures to fetch, if empty, then fetch all architectures
Architectures []string
// Should we download sources?
DownloadSources bool
// Should we download .udebs?
DownloadUdebs bool
// Meta-information about repository
Meta Stanza
// Last update date
@@ -57,20 +53,22 @@ type RemoteRepo struct {
ReleaseFiles map[string]utils.ChecksumInfo
// Filter for packages
Filter string
// Status marks state of repository (being updated, no action)
Status int
// WorkerPID is PID of the process modifying the mirror (if any)
WorkerPID int
// FilterWithDeps to include dependencies from filter query
FilterWithDeps bool
// SkipComponentCheck skips component list verification
SkipComponentCheck bool
// SkipArchitectureCheck skips architecture list verification
SkipArchitectureCheck bool
// Status marks state of repository (being updated, no action)
Status int
// WorkerPID is PID of the process modifying the mirror (if any)
WorkerPID int
// Should we download sources?
DownloadSources bool
// Should we download .udebs?
DownloadUdebs bool
// "Snapshot" of current list of packages
packageRefs *PackageRefList
// Temporary list of package refs
tempPackageRefs *PackageRefList
// Parsed archived root
archiveRootURL *url.URL
// Current list of packages (filled while updating mirror)
@@ -193,49 +191,49 @@ func (repo *RemoteRepo) CheckLock() error {
return nil
}
// ReleaseURL returns URL to Release* files in repo root
func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
// IndexesRootURL builds URL for various indexes
func (repo *RemoteRepo) IndexesRootURL() *url.URL {
var path *url.URL
if !repo.IsFlat() {
path = &url.URL{Path: fmt.Sprintf("dists/%s/%s", repo.Distribution, name)}
path = &url.URL{Path: fmt.Sprintf("dists/%s/", repo.Distribution)}
} else {
path = &url.URL{Path: filepath.Join(repo.Distribution, name)}
path = &url.URL{Path: repo.Distribution}
}
return repo.archiveRootURL.ResolveReference(path)
}
// FlatBinaryURL returns URL to Packages files for flat repo
func (repo *RemoteRepo) FlatBinaryURL() *url.URL {
path := &url.URL{Path: filepath.Join(repo.Distribution, "Packages")}
return repo.archiveRootURL.ResolveReference(path)
// ReleaseURL returns URL to Release* files in repo root
func (repo *RemoteRepo) ReleaseURL(name string) *url.URL {
return repo.IndexesRootURL().ResolveReference(&url.URL{Path: name})
}
// FlatSourcesURL returns URL to Sources files for flat repo
func (repo *RemoteRepo) FlatSourcesURL() *url.URL {
path := &url.URL{Path: filepath.Join(repo.Distribution, "Sources")}
return repo.archiveRootURL.ResolveReference(path)
// FlatBinaryPath returns path to Packages files for flat repo
func (repo *RemoteRepo) FlatBinaryPath() string {
return "Packages"
}
// BinaryURL returns URL of Packages files for given component and
// FlatSourcesPath returns path to Sources files for flat repo
func (repo *RemoteRepo) FlatSourcesPath() string {
return "Sources"
}
// BinaryPath returns path to Packages files for given component and
// architecture
func (repo *RemoteRepo) BinaryURL(component string, architecture string) *url.URL {
path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/binary-%s/Packages", repo.Distribution, component, architecture)}
return repo.archiveRootURL.ResolveReference(path)
func (repo *RemoteRepo) BinaryPath(component string, architecture string) string {
return fmt.Sprintf("%s/binary-%s/Packages", component, architecture)
}
// SourcesURL returns URL of Sources files for given component
func (repo *RemoteRepo) SourcesURL(component string) *url.URL {
path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/source/Sources", repo.Distribution, component)}
return repo.archiveRootURL.ResolveReference(path)
// SourcesPath returns path to Sources files for given component
func (repo *RemoteRepo) SourcesPath(component string) string {
return fmt.Sprintf("%s/source/Sources", component)
}
// UdebURL returns URL of Packages files for given component and
// UdebPath returns path of Packages files for given component and
// architecture
func (repo *RemoteRepo) UdebURL(component string, architecture string) *url.URL {
path := &url.URL{Path: fmt.Sprintf("dists/%s/%s/debian-installer/binary-%s/Packages", repo.Distribution, component, architecture)}
return repo.archiveRootURL.ResolveReference(path)
func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
return fmt.Sprintf("%s/debian-installer/binary-%s/Packages", component, architecture)
}
// PackageURL returns URL of package file relative to repository root
@@ -246,7 +244,7 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
}
// Fetch updates information about repository
func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error {
func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
var (
release, inrelease, releasesig *os.File
err error
@@ -292,7 +290,7 @@ func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier utils.Verifier) error
return err
}
err = verifier.VerifyDetachedSignature(releasesig, release)
err = verifier.VerifyDetachedSignature(releasesig, release, true)
if err != nil {
return err
}
@@ -316,7 +314,7 @@ ok:
architectures := strings.Split(stanza["Architectures"], " ")
sort.Strings(architectures)
// "source" architecture is never present, despite Release file claims
architectures = utils.StrSlicesSubstract(architectures, []string{"source"})
architectures = utils.StrSlicesSubstract(architectures, []string{ArchitectureSource})
if len(repo.Architectures) == 0 {
repo.Architectures = architectures
} else if !repo.SkipArchitectureCheck {
@@ -331,9 +329,7 @@ ok:
if strings.Contains(repo.Distribution, "/") {
distributionLast := path.Base(repo.Distribution) + "/"
for i := range components {
if strings.HasPrefix(components[i], distributionLast) {
components[i] = components[i][len(distributionLast):]
}
components[i] = strings.TrimPrefix(components[i], distributionLast)
}
}
if len(repo.Components) == 0 {
@@ -411,30 +407,30 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
repo.packageList = NewPackageList()
// Download and parse all Packages & Source files
packagesURLs := [][]string{}
packagesPaths := [][]string{}
if repo.IsFlat() {
packagesURLs = append(packagesURLs, []string{repo.FlatBinaryURL().String(), "binary"})
packagesPaths = append(packagesPaths, []string{repo.FlatBinaryPath(), PackageTypeBinary})
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.FlatSourcesURL().String(), "source"})
packagesPaths = append(packagesPaths, []string{repo.FlatSourcesPath(), PackageTypeSource})
}
} else {
for _, component := range repo.Components {
for _, architecture := range repo.Architectures {
packagesURLs = append(packagesURLs, []string{repo.BinaryURL(component, architecture).String(), "binary"})
packagesPaths = append(packagesPaths, []string{repo.BinaryPath(component, architecture), PackageTypeBinary})
if repo.DownloadUdebs {
packagesURLs = append(packagesURLs, []string{repo.UdebURL(component, architecture).String(), "udeb"})
packagesPaths = append(packagesPaths, []string{repo.UdebPath(component, architecture), PackageTypeUdeb})
}
}
if repo.DownloadSources {
packagesURLs = append(packagesURLs, []string{repo.SourcesURL(component).String(), "source"})
packagesPaths = append(packagesPaths, []string{repo.SourcesPath(component), PackageTypeSource})
}
}
}
for _, info := range packagesURLs {
url, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, url, repo.ReleaseFiles, ignoreMismatch, maxTries)
for _, info := range packagesPaths {
path, kind := info[0], info[1]
packagesReader, packagesFile, err := http.DownloadTryCompression(d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch, maxTries)
if err != nil {
return err
}
@@ -459,11 +455,11 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
var p *Package
if kind == "binary" {
if kind == PackageTypeBinary {
p = NewPackageFromControlFile(stanza)
} else if kind == "udeb" {
} else if kind == PackageTypeUdeb {
p = NewUdebPackageFromControlFile(stanza)
} else if kind == "source" {
} else if kind == PackageTypeSource {
p, err = NewSourcePackageFromControlFile(stanza)
if err != nil {
return err
@@ -477,11 +473,6 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
return err
}
}
err = collectionFactory.PackageCollection().Update(p)
if err != nil {
return err
}
}
progress.ShutdownBar()
@@ -491,14 +482,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
}
// ApplyFilter applies filtering to already built PackageList
func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery) (oldLen, newLen int, err error) {
func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQuery, progress aptly.Progress) (oldLen, newLen int, err error) {
repo.packageList.PrepareIndex()
emptyList := NewPackageList()
emptyList.PrepareIndex()
oldLen = repo.packageList.Len()
repo.packageList, err = repo.packageList.Filter([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures)
repo.packageList, err = repo.packageList.FilterWithProgress([]PackageQuery{filterQuery}, repo.FilterWithDeps, emptyList, dependencyOptions, repo.Architectures, progress)
if repo.packageList != nil {
newLen = repo.packageList.Len()
}
@@ -506,24 +497,40 @@ func (repo *RemoteRepo) ApplyFilter(dependencyOptions int, filterQuery PackageQu
}
// BuildDownloadQueue builds queue, discards current PackageList
func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue []PackageDownloadTask, downloadSize int64, err error) {
func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool, packageCollection *PackageCollection, checksumStorage aptly.ChecksumStorage, skipExistingPackages bool) (queue []PackageDownloadTask, downloadSize int64, err error) {
queue = make([]PackageDownloadTask, 0, repo.packageList.Len())
seen := make(map[string]struct{}, repo.packageList.Len())
seen := make(map[string]int, repo.packageList.Len())
err = repo.packageList.ForEach(func(p *Package) error {
list, err2 := p.DownloadList(packagePool)
if repo.packageRefs != nil && skipExistingPackages {
if repo.packageRefs.Has(p) {
// skip this package, but load checksums/files from package in DB
var prevP *Package
prevP, err = packageCollection.ByKey(p.Key(""))
if err != nil {
return err
}
p.UpdateFiles(prevP.Files())
return nil
}
}
list, err2 := p.DownloadList(packagePool, checksumStorage)
if err2 != nil {
return err2
}
p.files = nil
for _, task := range list {
key := task.RepoURI + "-" + task.DestinationPath
_, found := seen[key]
key := task.File.DownloadURL()
idx, found := seen[key]
if !found {
queue = append(queue, task)
downloadSize += task.Checksums.Size
seen[key] = struct{}{}
downloadSize += task.File.Checksums.Size
seen[key] = len(queue) - 1
} else {
// hook up the task to duplicate entry already on the list
queue[idx].Additional = append(queue[idx].Additional, task)
}
}
@@ -533,17 +540,39 @@ func (repo *RemoteRepo) BuildDownloadQueue(packagePool aptly.PackagePool) (queue
return
}
repo.tempPackageRefs = NewPackageRefListFromPackageList(repo.packageList)
// free up package list, we don't need it after this point
repo.packageList = nil
return
}
// FinalizeDownload swaps for final value of package refs
func (repo *RemoteRepo) FinalizeDownload() {
func (repo *RemoteRepo) FinalizeDownload(collectionFactory *CollectionFactory, progress aptly.Progress) error {
repo.LastDownloadDate = time.Now()
repo.packageRefs = repo.tempPackageRefs
if progress != nil {
progress.InitBar(int64(repo.packageList.Len()), true)
}
var i int
// update all the packages in collection
err := repo.packageList.ForEach(func(p *Package) error {
i++
if progress != nil {
progress.SetBar(i)
}
// download process might have updated checksums
p.UpdateFiles(p.Files())
return collectionFactory.PackageCollection().Update(p)
})
repo.packageRefs = NewPackageRefListFromPackageList(repo.packageList)
if progress != nil {
progress.ShutdownBar()
}
repo.packageList = nil
return err
}
// Encode does msgpack encoding of RemoteRepo