vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-12 03:21:33 +00:00
Code Issues Projects Releases Wiki Activity

Implement 'legacy' Contents indexes to match Ubuntu <=16.04

Browse Source 
Another index is created which unifies data for all the components.

This certainly requires more resources as we have to build yet another
index.
This commit is contained in:
Andrey Smirnov
2018-04-11 00:57:15 +03:00
parent 5f96abc271
commit 5b85522400
9 changed files with 86 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Gopkg.lock generated
View File

@@ -56,8 +56,8 @@
"service/s3",
"service/sts"
]
revision = "7f68df2a5baf19398d17def23a514a6e617e5937"
version = "v1.13.28"
revision = "a72204b9bf8d48230ee0fe8995613b394c66f2da"
version = "v1.13.31"
[[projects]]
name = "github.com/cheggaaa/pb"

6
deb/contents.go
View File

@@ -6,7 +6,6 @@ import (
"fmt"
"io"
"github.com/smira/aptly/aptly"
"github.com/smira/aptly/database"
"github.com/smira/go-uuid/uuid"
)
@@ -26,10 +25,7 @@ func NewContentsIndex(db database.Storage) *ContentsIndex {
}
// Push adds package to contents index, calculating package contents as required
func (index *ContentsIndex) Push(p *Package, packagePool aptly.PackagePool, progress aptly.Progress) error {
contents := p.Contents(packagePool, progress)
qualifiedName := []byte(p.QualifiedName())
func (index *ContentsIndex) Push(qualifiedName []byte, contents []string) error {
for _, path := range contents {
// for performance reasons we only write to leveldb during push.
// merging of qualified names per path will be done in WriteTo

31
deb/index_files.go
View File

@@ -329,6 +329,37 @@ func (files *indexFiles) ContentsIndex(component, arch string, udeb bool) *index
return file
}
func (files *indexFiles) LegacyContentsIndex(arch string, udeb bool) *indexFile {
if arch == ArchitectureSource {
udeb = false
}
key := fmt.Sprintf("lci-%s-%v", arch, udeb)
file, ok := files.indexes[key]
if !ok {
var relativePath string
if udeb {
relativePath = fmt.Sprintf("Contents-udeb-%s", arch)
} else {
relativePath = fmt.Sprintf("Contents-%s", arch)
}
file = &indexFile{
parent: files,
discardable: true,
compressable: true,
onlyGzip: true,
signable: false,
acquireByHash: files.acquireByHash,
relativePath: relativePath,
}
files.indexes[key] = file
}
return file
}
func (files *indexFiles) ReleaseFile() *indexFile {
return &indexFile{
parent: files,

38
deb/publish.go
View File

@@ -562,6 +562,8 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
legacyContentIndexes := map[string]*ContentsIndex{}
for component, list := range lists {
hadUdebs := false
@@ -612,15 +614,19 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
if !p.SkipContents {
key := fmt.Sprintf("%s-%v", arch, pkg.IsUdeb)
qualifiedName := []byte(pkg.QualifiedName())
contents := pkg.Contents(packagePool, progress)
contentIndex := contentIndexes[key]
for _, contentIndexesMap := range []map[string]*ContentsIndex{contentIndexes, legacyContentIndexes} {
contentIndex := contentIndexesMap[key]
if contentIndex == nil {
contentIndex = NewContentsIndex(tempDB)
contentIndexes[key] = contentIndex
if contentIndex == nil {
contentIndex = NewContentsIndex(tempDB)
contentIndexesMap[key] = contentIndex
}
contentIndex.Push(qualifiedName, contents)
}
contentIndex.Push(pkg, packagePool, progress)
}
bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb).BufWriter()
@@ -712,6 +718,26 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
}
for _, arch := range p.Architectures {
for _, udeb := range []bool{true, false} {
index := legacyContentIndexes[fmt.Sprintf("%s-%v", arch, udeb)]
if index == nil || index.Empty() {
continue
}
var bufWriter *bufio.Writer
bufWriter, err = indexes.LegacyContentsIndex(arch, udeb).BufWriter()
if err != nil {
return fmt.Errorf("unable to generate contents index: %v", err)
}
_, err = index.WriteTo(bufWriter)
if err != nil {
return fmt.Errorf("unable to generate contents index: %v", err)
}
}
}
if progress != nil {
progress.Printf("Finalizing metadata files...\n")
}

3
system/t06_publish/PublishRepo1Test_contents_i386_legacy Normal file
View File

@@ -0,0 +1,3 @@
FILE LOCATION
usr/share/doc/libboost-program-options-dev/changelog.gz libdevel/libboost-program-options-dev
usr/share/doc/libboost-program-options-dev/copyright libdevel/libboost-program-options-dev

6
system/t06_publish/repo.py
View File

@@ -54,6 +54,7 @@ class PublishRepo1Test(BaseTest):
self.check_file_contents('public/dists/maverick/main/source/Sources', 'sources', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
self.check_file_contents('public/dists/maverick/main/binary-i386/Packages', 'binary', match_prepare=lambda s: "\n".join(sorted(s.split("\n"))))
self.check_file_contents('public/dists/maverick/main/Contents-i386.gz', 'contents_i386', match_prepare=ungzip_if_required)
self.check_file_contents('public/dists/maverick/Contents-i386.gz', 'contents_i386_legacy', match_prepare=ungzip_if_required)
# verify signatures
self.run_cmd(["gpg", "--no-auto-check-trustdb", "--keyring", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", "aptly.pub"),
@@ -92,7 +93,8 @@ class PublishRepo1Test(BaseTest):
if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2',
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']):
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz',
'Contents-i386.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -492,7 +494,7 @@ class PublishRepo17Test(BaseTest):
'contrib/source/Sources', 'contrib/source/Sources.gz', 'contrib/source/Sources.bz2',
'main/source/Release', 'contrib/source/Release',
'main/binary-i386/Release', 'contrib/binary-i386/Release',
'main/Contents-i386.gz']):
'main/Contents-i386.gz', 'Contents-i386.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))

6
system/t06_publish/snapshot.py
View File

@@ -104,7 +104,7 @@ class PublishSnapshot1Test(BaseTest):
if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz',
'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2',
'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
'main/Contents-i386.gz']):
'main/Contents-i386.gz', 'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -754,7 +754,8 @@ class PublishSnapshot26Test(BaseTest):
'contrib/source/Sources', 'contrib/source/Sources.gz', 'contrib/source/Sources.bz2',
'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/source/Release',
'contrib/binary-amd64/Release', 'contrib/binary-i386/Release', 'contrib/source/Release',
'contrib/Contents-i386.gz', 'main/Contents-i386.gz', 'main/Contents-amd64.gz']):
'contrib/Contents-i386.gz', 'main/Contents-i386.gz', 'main/Contents-amd64.gz',
'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -975,6 +976,7 @@ class PublishSnapshot35Test(BaseTest):
pathsExepcted.add("main/%sbinary-%s/Packages%s" % (udeb, arch, ext))
pathsExepcted.add("main/Contents-%s%s.gz" % ("udeb-" if udeb != "" else "", arch))
pathsExepcted.add("Contents-%s%s.gz" % ("udeb-" if udeb != "" else "", arch))
pathsExepcted.add("main/%sbinary-%s/Release" % (udeb, arch))

6
system/t06_publish/switch.py
View File

@@ -86,7 +86,7 @@ class PublishSwitch1Test(BaseTest):
if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz',
'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2',
'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
'main/Contents-i386.gz']):
'main/Contents-i386.gz', 'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -354,7 +354,7 @@ class PublishSwitch8Test(BaseTest):
'a/binary-i386/Release', 'b/binary-i386/Release', 'c/binary-i386/Release',
'a/source/Release', 'b/source/Release', 'c/source/Release',
'b/Contents-amd64.gz', 'c/Contents-i386.gz', 'a/Contents-i386.gz',
'a/Contents-amd64.gz', 'b/Contents-i386.gz']):
'a/Contents-amd64.gz', 'b/Contents-i386.gz', 'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -531,5 +531,5 @@ class PublishSwitch14Test(BaseTest):
if pathsSeen != set(['main/binary-amd64/Packages', 'main/binary-i386/Packages', 'main/binary-i386/Packages.gz',
'main/binary-amd64/Packages.gz', 'main/binary-amd64/Packages.bz2', 'main/binary-i386/Packages.bz2',
'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
'main/Contents-i386.gz']):
'main/Contents-i386.gz', 'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))

8
system/t06_publish/update.py
View File

@@ -31,6 +31,7 @@ class PublishUpdate1Test(BaseTest):
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/Contents-i386.gz')
self.check_exists('public/dists/maverick/main/Contents-i386.gz')
self.check_exists('public/dists/maverick/main/source/Sources')
self.check_exists('public/dists/maverick/main/source/Sources.gz')
@@ -84,7 +85,8 @@ class PublishUpdate1Test(BaseTest):
if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2',
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']):
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz',
'Contents-i386.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
@@ -370,6 +372,7 @@ class PublishUpdate12Test(BaseTest):
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/Contents-i386.gz')
self.check_exists('public/dists/maverick/main/Contents-i386.gz')
self.check_exists('public/dists/maverick/main/source/Sources')
self.check_exists('public/dists/maverick/main/source/Sources.gz')
@@ -423,5 +426,6 @@ class PublishUpdate12Test(BaseTest):
if pathsSeen != set(['main/binary-i386/Packages', 'main/binary-i386/Packages.bz2', 'main/binary-i386/Packages.gz',
'main/source/Sources', 'main/source/Sources.gz', 'main/source/Sources.bz2',
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz']):
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz',
'Contents-i386.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))