mirror of
https://github.com/aptly-dev/aptly.git
synced 2026-06-07 05:42:42 +00:00
Allow disabling bzip2 compression for index files
Using bzip2 generates smaller index files (roughly 20% smaller Packages
files) but it comes with a big performance penalty. When publishing a
debian mirror snapshot (amd64, arm64, armhf, source) without contents
skipping bzip speeds things up around 1.8 times.
```
$ hyperfine -w 1 -L skip-bz2 true,false -m 3 -p "aptly -config aptly.conf publish drop bullseye || true" "aptly -config aptly.conf publish snapshot --skip-bz2={skip-bz2} --skip-contents --skip-signing bullseye"
Benchmark 1: aptly -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye
Time (mean ± σ): 35.567 s ± 0.307 s [User: 39.366 s, System: 10.075 s]
Range (min … max): 35.311 s … 35.907 s 3 runs
Benchmark 2: aptly -config aptly.conf publish snapshot --skip-bz2=false --skip-contents --skip-signing bullseye
Time (mean ± σ): 64.740 s ± 0.135 s [User: 68.565 s, System: 10.129 s]
Range (min … max): 64.596 s … 64.862 s 3 runs
Summary
'aptly -config aptly.conf publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ran
1.82 ± 0.02 times faster than 'aptly -config aptly.conf publish snapshot --skip-bz2=false --skip-contents --skip-signing bullseye'
```
Allow skipping bz2 creation for setups where faster publishing is more
important then Package file size.
Signed-off-by: Sjoerd Simons <sjoerd@collabora.com>
This commit is contained in:
committed by
Benj Fassbind
parent
2aca913e92
commit
f61514edaf
+10
-4
@@ -22,6 +22,7 @@ type indexFiles struct {
|
||||
suffix string
|
||||
indexes map[string]*indexFile
|
||||
acquireByHash bool
|
||||
skipBz2 bool
|
||||
}
|
||||
|
||||
type indexFile struct {
|
||||
@@ -68,7 +69,7 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
|
||||
}
|
||||
|
||||
if file.compressable {
|
||||
err = utils.CompressFile(file.tempFile, file.onlyGzip)
|
||||
err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
|
||||
if err != nil {
|
||||
file.tempFile.Close()
|
||||
return fmt.Errorf("unable to compress index file: %s", err)
|
||||
@@ -80,11 +81,15 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
|
||||
exts := []string{""}
|
||||
cksumExts := exts
|
||||
if file.compressable {
|
||||
exts = append(exts, ".gz", ".bz2")
|
||||
cksumExts = exts
|
||||
if file.onlyGzip {
|
||||
exts = []string{".gz"}
|
||||
cksumExts = []string{"", ".gz"}
|
||||
} else {
|
||||
exts = append(exts, ".gz")
|
||||
if !file.parent.skipBz2 {
|
||||
exts = append(exts, ".bz2")
|
||||
}
|
||||
cksumExts = exts
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,7 +234,7 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
|
||||
return nil
|
||||
}
|
||||
|
||||
func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool) *indexFiles {
|
||||
func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, suffix string, acquireByHash bool, skipBz2 bool) *indexFiles {
|
||||
return &indexFiles{
|
||||
publishedStorage: publishedStorage,
|
||||
basePath: basePath,
|
||||
@@ -239,6 +244,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
|
||||
suffix: suffix,
|
||||
indexes: make(map[string]*indexFile),
|
||||
acquireByHash: acquireByHash,
|
||||
skipBz2: skipBz2,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+4
-1
@@ -62,6 +62,9 @@ type PublishedRepo struct {
|
||||
// Skip contents generation
|
||||
SkipContents bool
|
||||
|
||||
// Skip bz2 compression for index files
|
||||
SkipBz2 bool
|
||||
|
||||
// True if repo is being re-published
|
||||
rePublishing bool
|
||||
|
||||
@@ -585,7 +588,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash)
|
||||
indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash, p.SkipBz2)
|
||||
|
||||
legacyContentIndexes := map[string]*ContentsIndex{}
|
||||
var count int64
|
||||
|
||||
Reference in New Issue
Block a user