vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-05-06 22:18:28 +00:00
Code Issues Projects Releases Wiki Activity

Allow disabling bzip2 compression for index files

Browse Source 
Using bzip2 generates smaller index files (roughly 20% smaller Packages
files) but it comes with a big performance penalty.  When publishing a
debian mirror snapshot (amd64, arm64, armhf, source) without contents
skipping bzip speeds things up around 1.8 times.

```
$ hyperfine -w 1 -L skip-bz2 true,false  -m 3 -p "aptly -config aptly.conf publish drop bullseye || true" "aptly -config aptly.conf  publish snapshot  --skip-bz2={skip-bz2} --skip-contents --skip-signing bullseye"
Benchmark 1: aptly -config aptly.conf  publish snapshot  --skip-bz2=true --skip-contents --skip-signing bullseye
  Time (mean ± σ):     35.567 s ±  0.307 s    [User: 39.366 s, System: 10.075 s]
  Range (min … max):   35.311 s … 35.907 s    3 runs

Benchmark 2: aptly -config aptly.conf  publish snapshot  --skip-bz2=false --skip-contents --skip-signing bullseye
  Time (mean ± σ):     64.740 s ±  0.135 s    [User: 68.565 s, System: 10.129 s]
  Range (min … max):   64.596 s … 64.862 s    3 runs

Summary
  'aptly -config aptly.conf  publish snapshot  --skip-bz2=true --skip-contents --skip-signing bullseye' ran
    1.82 ± 0.02 times faster than 'aptly -config aptly.conf  publish snapshot  --skip-bz2=false --skip-contents --skip-signing bullseye'
```

Allow skipping bz2 creation for setups where faster publishing is more
important then Package file size.

Signed-off-by: Sjoerd Simons <sjoerd@collabora.com>
This commit is contained in:
Sjoerd Simons
2022-06-18 08:29:21 +02:00
committed by Benj Fassbind
parent 2aca913e92
commit f61514edaf
21 changed files with 204 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
system/t06_publish/PublishRepo33Test_gold
+14
View File
@@ -0,0 +1,14 @@
Loading packages...
Generating metadata files and linking package files...
Finalizing metadata files...
Signing file 'Release' with gpg, please enter your passphrase when prompted:
Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
Local repo local-repo has been successfully published.
Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
Now you can add following line to apt sources:
deb http://your-server/ maverick main
deb-src http://your-server/ maverick main
Don't forget to add your GPG key to apt with apt-key.
You can also use `aptly serve` to publish your repositories over HTTP quickly.
system/t06_publish/PublishSnapshot40Test_gold
+13
View File
@@ -0,0 +1,13 @@
Loading packages...
Generating metadata files and linking package files...
Finalizing metadata files...
Signing file 'Release' with gpg, please enter your passphrase when prompted:
Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
Snapshot snap40 has been successfully published.
Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
Now you can add following line to apt sources:
deb http://your-server/ maverick main
Don't forget to add your GPG key to apt with apt-key.
You can also use `aptly serve` to publish your repositories over HTTP quickly.
system/t06_publish/PublishSwitch15Test_gold
+8
View File
@@ -0,0 +1,8 @@
Loading packages...
Generating metadata files and linking package files...
Finalizing metadata files...
Signing file 'Release' with gpg, please enter your passphrase when prompted:
Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
Cleaning up prefix "." components main...
Publish for snapshot ./maverick (origin: LP-PPA-gladky-anton-gnuplot) [amd64, i386] publishes {main: [snap3]: Pulled into 'snap2' with 'snap1' as source, pull request was: 'gnuplot-x11'} has been successfully switched to new snapshot.
system/t06_publish/PublishUpdate13Test_gold
+8
View File
@@ -0,0 +1,8 @@
Loading packages...
Generating metadata files and linking package files...
Finalizing metadata files...
Signing file 'Release' with gpg, please enter your passphrase when prompted:
Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
Cleaning up prefix "." components main...
Publish for local repo ./maverick [i386, source] publishes {main: [local-repo]} has been successfully updated.
system/t06_publish/repo.py
+28 -2
View File
@@ -29,8 +29,7 @@ class PublishRepo1Test(BaseTest):
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_exists(
'public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/Contents-i386.gz')
self.check_exists('public/dists/maverick/main/source/Sources')
self.check_exists('public/dists/maverick/main/source/Sources.gz')
@@ -862,3 +861,30 @@ class PublishRepo32Test(BaseTest):
"--verify", os.path.join(
os.environ["HOME"], ".aptly", 'public/dists/maverick/Release.gpg'),
os.path.join(os.environ["HOME"], ".aptly", 'public/dists/maverick/Release')])
class PublishRepo33Test(BaseTest):
"""
publish repo: -skip-bz2
"""
fixtureCmds = [
"aptly repo create local-repo",
"aptly repo add local-repo ${files} ${udebs}",
]
runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-bz2 local-repo"
gold_processor = BaseTest.expand_environ
def check(self):
super(PublishRepo33Test, self).check()
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/main/binary-i386/Release')
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/binary-amd64/Release')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-amd64/Packages.bz2')
system/t06_publish/snapshot.py
+29
View File
@@ -1210,3 +1210,32 @@ class PublishSnapshot39Test(BaseTest):
'contents_i386', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False)
self.check_file_contents('public/dists/maverick/main/Contents-amd64.gz',
'contents_amd64', match_prepare=ungzip_if_required, mode='b', ensure_utf8=False)
class PublishSnapshot40Test(BaseTest):
"""
publish snapshot: -skip-bz2
"""
fixtureDB = True
fixturePool = True
fixtureCmds = [
"aptly snapshot create snap40 from mirror gnuplot-maverick",
]
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-bz2 snap40"
gold_processor = BaseTest.expand_environ
def check(self):
super(PublishSnapshot40Test, self).check()
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/Release.gpg')
self.check_exists('public/dists/maverick/main/binary-i386/Release')
self.check_exists('public/dists/maverick/main/binary-amd64/Release')
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-amd64/Packages.bz2')
system/t06_publish/switch.py
+29
View File
@@ -545,3 +545,32 @@ class PublishSwitch14Test(BaseTest):
'main/binary-amd64/Release', 'main/binary-i386/Release', 'main/Contents-amd64.gz',
'main/Contents-i386.gz', 'Contents-i386.gz', 'Contents-amd64.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
class PublishSwitch15Test(BaseTest):
"""
publish switch: -skip-bz2
"""
fixtureDB = True
fixturePool = True
fixtureCmds = [
"aptly snapshot create snap1 from mirror gnuplot-maverick",
"aptly snapshot create snap2 empty",
"aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-bz2 snap1",
]
runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick snap3"
gold_processor = BaseTest.expand_environ
def check(self):
super(PublishSwitch15Test, self).check()
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages')
self.check_exists('public/dists/maverick/main/binary-amd64/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-amd64/Packages.bz2')
system/t06_publish/update.py
+25
View File
@@ -437,3 +437,28 @@ class PublishUpdate12Test(BaseTest):
'main/binary-i386/Release', 'main/source/Release', 'main/Contents-i386.gz',
'Contents-i386.gz']):
raise Exception("path seen wrong: %r" % (pathsSeen, ))
class PublishUpdate13Test(BaseTest):
"""
publish update: -skip-bz2
"""
fixtureCmds = [
"aptly repo create local-repo",
"aptly repo add local-repo ${files}/",
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick -skip-bz2 local-repo",
"aptly repo remove local-repo pyspi"
]
runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-bz2 maverick"
gold_processor = BaseTest.expand_environ
def check(self):
super(PublishUpdate13Test, self).check()
self.check_exists('public/dists/maverick/InRelease')
self.check_exists('public/dists/maverick/Release')
self.check_exists('public/dists/maverick/Release.gpg')
self.check_exists('public/dists/maverick/main/binary-i386/Packages')
self.check_exists('public/dists/maverick/main/binary-i386/Packages.gz')
self.check_not_exists('public/dists/maverick/main/binary-i386/Packages.bz2')