vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-01-11 03:11:50 +00:00
Code Issues Projects Releases Wiki Activity

system-tests: improve sorted compare

Browse Source 
sort both aptly output and gold file. output original output for
debugging on failure.

* Makefile: enable CAPTURE=1 env variable for capturing gold files
* docker-system-test: use AWS env vars for S3 tests
* fix system tests timing issue with order of gpg logs in publish tests
This commit is contained in:
André Roth
2025-04-21 14:13:37 +02:00
parent c07bf2b108
commit 22bc2f9d0f
5 changed files with 66 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Makefile
View File

@@ -7,8 +7,10 @@ COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH)
# Uncomment to update system test gold files
# CAPTURE := "--capture"
# export CAPUTRE=1 for regenrating test gold files
ifeq ($(CAPTURE),1)
CAPTURE_ARG := --capture
endif
help: ## Print this help
@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -50,7 +52,7 @@ swagger-install:
echo "// @version $(VERSION)" >> docs/swagger.conf
azurite-start:
azurite -l /tmp/aptly-azurite & \
azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
echo $$! > ~/.azurite.pid
azurite-stop:
@@ -102,7 +104,7 @@ system-test: prepare swagger etcd-install ## Run system tests
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests
PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST)
bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -186,6 +188,8 @@ docker-system-test: ## Run system tests in docker container (add TEST=t04_mirro
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
system-test TEST=$(TEST) \
azurite-stop

29
system/lib.py
View File

@@ -289,8 +289,6 @@ class BaseTest(object):
def run(self):
output = self.run_cmd(self.runCmd, self.expectedCode)
if self.sortOutput:
output = self.sort_lines(output)
self.output = self.output_processor(output)
def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None):
@@ -493,24 +491,29 @@ class BaseTest(object):
return a.decode('utf-8')
return a
def verify_match(self, a, b, match_prepare=None, ensure_utf8=True):
def verify_match(self, gold, orig, match_prepare=None, ensure_utf8=True):
output = orig
if ensure_utf8:
a = self.ensure_utf8(a)
b = self.ensure_utf8(b)
gold = self.ensure_utf8(gold)
output = self.ensure_utf8(output)
if match_prepare is not None:
a = match_prepare(a)
b = match_prepare(b)
gold = match_prepare(gold)
output = match_prepare(output)
# sort if requested
if self.sortOutput:
gold = self.sort_lines(gold)
output = self.sort_lines(output)
# strip trailing whitespace and newlines
a = a.strip()
b = b.strip()
gold = gold.strip()
output = output.strip()
if a != b:
if gold != output:
diff = "".join(difflib.unified_diff(
[l + "\n" for l in a.split("\n")], [l + "\n" for l in b.split("\n")]))
raise Exception("content doesn't match:\n" + diff + "\n")
[l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")]))
raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n")
check = check_output

1
system/t06_publish/PublishSnapshot38Test_gold
View File

@@ -12,3 +12,4 @@ Now you can add following line to apt sources:
Don't forget to add your GPG key to apt with apt-key.
You can also use `aptly serve` to publish your repositories over HTTP quickly.

6
system/t06_publish/s3.py
View File

@@ -14,6 +14,7 @@ class S3Publish1Test(S3Test):
"aptly repo add local-repo ${files}",
"aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386",
]
sortOutput = True
runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:"
def check(self):
@@ -53,6 +54,7 @@ class S3Publish2Test(S3Test):
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:",
"aptly repo remove local-repo pyspi"
]
sortOutput = True
runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:"
def check(self):
@@ -93,6 +95,7 @@ class S3Publish3Test(S3Test):
"aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1 s3:test1:",
]
sortOutput = True
runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1: snap3"
def check(self):
@@ -129,6 +132,7 @@ class S3Publish4Test(S3Test):
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=xyz local-repo s3:test1:",
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:prefix",
]
sortOutput = True
runCmd = "aptly publish list"
@@ -144,6 +148,7 @@ class S3Publish5Test(S3Test):
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 s3:test1:",
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 s3:test1:",
]
sortOutput = True
runCmd = "aptly publish drop sq2 s3:test1:"
def check(self):
@@ -172,6 +177,7 @@ class S3Publish6Test(S3Test):
"aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:",
"aptly repo remove local-repo pyspi"
]
sortOutput = True
runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:"
def check(self):

35
system/t06_publish/snapshot.py
View File

@@ -22,6 +22,7 @@ class PublishSnapshot1Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap1 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1"
gold_processor = BaseTest.expand_environ
@@ -131,6 +132,7 @@ class PublishSnapshot2Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap2 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2"
gold_processor = BaseTest.expand_environ
@@ -168,6 +170,7 @@ class PublishSnapshot3Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap3 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -component=contrib snap3"
gold_processor = BaseTest.expand_environ
@@ -208,6 +211,7 @@ class PublishSnapshot4Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap4 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly -architectures=i386 publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap4"
gold_processor = BaseTest.expand_environ
@@ -247,6 +251,7 @@ class PublishSnapshot5Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap5 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira"
gold_processor = BaseTest.expand_environ
@@ -294,6 +299,7 @@ class PublishSnapshot6Test(BaseTest):
"aptly snapshot create snap2 from mirror wheezy-main",
"aptly snapshot merge snap6 snap2 snap"
]
sortOutput = True
runCmd = "aptly publish snapshot snap6"
expectedCode = 1
@@ -308,6 +314,7 @@ class PublishSnapshot7Test(BaseTest):
"aptly snapshot create snap7 from mirror gnuplot-maverick",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap7",
]
sortOutput = True
runCmd = "aptly publish snapshot snap7"
expectedCode = 1
@@ -322,6 +329,7 @@ class PublishSnapshot8Test(BaseTest):
"aptly snapshot create snap8 from mirror gnuplot-maverick",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap8 ./ppa",
]
sortOutput = True
runCmd = "aptly publish snapshot snap8 ppa"
expectedCode = 1
@@ -334,6 +342,7 @@ class PublishSnapshot9Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap9 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot snap9 ppa/dists/la"
expectedCode = 1
@@ -346,6 +355,7 @@ class PublishSnapshot10Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap10 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot snap10 ppa/pool/la"
expectedCode = 1
@@ -358,6 +368,7 @@ class PublishSnapshot11Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap11 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot snap11 ../la"
expectedCode = 1
@@ -367,6 +378,7 @@ class PublishSnapshot12Test(BaseTest):
publish snapshot: no snapshot
"""
fixtureDB = True
sortOutput = True
runCmd = "aptly publish snapshot snap12"
expectedCode = 1
@@ -465,6 +477,7 @@ class PublishSnapshot16Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap16 from mirror gnuplot-maverick-src",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap16"
gold_processor = BaseTest.expand_environ
@@ -524,6 +537,7 @@ class PublishSnapshot17Test(BaseTest):
"aptly repo add local-repo ${files}",
"aptly snapshot create snap17 from repo local-repo",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap17"
gold_processor = BaseTest.expand_environ
@@ -578,6 +592,7 @@ class PublishSnapshot18Test(BaseTest):
"aptly repo add repo1 ${files}",
"aptly snapshot create snap1 from repo repo1",
]
sortOutput = True
runCmd = "aptly publish snapshot snap1"
expectedCode = 1
@@ -682,6 +697,7 @@ class PublishSnapshot24Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap24 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -origin=aptly24 -notautomatic=yes -butautomaticupgrades=yes snap24"
gold_processor = BaseTest.expand_environ
@@ -701,6 +717,7 @@ class PublishSnapshot25Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap25 empty",
]
sortOutput = True
runCmd = "aptly publish snapshot -architectures=amd64 --distribution=maverick -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap25"
gold_processor = BaseTest.expand_environ
@@ -736,6 +753,7 @@ class PublishSnapshot26Test(BaseTest):
"aptly repo add local-repo ${files}",
"aptly snapshot create snap26.2 from repo local-repo",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=main,contrib snap26.1 snap26.2"
gold_processor = BaseTest.expand_environ
@@ -864,6 +882,7 @@ class PublishSnapshot27Test(BaseTest):
"aptly repo add local-repo ${files}",
"aptly snapshot create snap27.2 from repo local-repo",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=, snap27.1 snap27.2"
gold_processor = BaseTest.expand_environ
@@ -879,6 +898,7 @@ class PublishSnapshot28Test(BaseTest):
"aptly repo add local-repo ${files}",
"aptly snapshot create snap28.2 from repo local-repo",
]
sortOutput = True
runCmd = "aptly publish snapshot -component=, snap28.1 snap28.2"
expectedCode = 1
@@ -891,6 +911,7 @@ class PublishSnapshot29Test(BaseTest):
"aptly snapshot create snap29.1 empty",
"aptly snapshot create snap29.2 empty",
]
sortOutput = True
runCmd = "aptly publish snapshot -component=b,b snap29.1 snap29.2"
expectedCode = 1
@@ -906,6 +927,7 @@ class PublishSnapshot30Test(BaseTest):
"aptly repo add local-repo ${files}",
"aptly snapshot create snap30.2 from repo local-repo",
]
sortOutput = True
runCmd = "aptly publish snapshot -component=main,contrib snap30.1 snap30.2"
expectedCode = 1
@@ -917,6 +939,7 @@ class PublishSnapshot31Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap31.1 empty",
]
sortOutput = True
runCmd = "aptly publish snapshot -component=main,contrib snap31.1 snap31.2"
expectedCode = 1
@@ -928,6 +951,7 @@ class PublishSnapshot32Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap32.1 empty",
]
sortOutput = True
runCmd = "aptly publish snapshot -component=main,contrib snap32.1"
expectedCode = 2
@@ -948,6 +972,7 @@ class PublishSnapshot33Test(BaseTest):
"aptly snapshot create snap2 from repo local-repo2",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2"
expectedCode = 1
gold_processor = BaseTest.expand_environ
@@ -966,6 +991,7 @@ class PublishSnapshot34Test(BaseTest):
"aptly snapshot create snap2 from repo local-repo2",
"aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1",
]
sortOutput = True
runCmd = "aptly publish snapshot -force-overwrite -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2"
gold_processor = BaseTest.expand_environ
@@ -987,6 +1013,7 @@ class PublishSnapshot35Test(BaseTest):
"aptly mirror update -keyring=aptlytest.gpg stretch",
"aptly snapshot create stretch from mirror stretch",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch"
gold_processor = BaseTest.expand_environ
@@ -1118,6 +1145,7 @@ class PublishSnapshot36Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap36 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-contents snap36"
gold_processor = BaseTest.expand_environ
@@ -1145,6 +1173,7 @@ class PublishSnapshot37Test(BaseTest):
"aptly mirror update -keyring=aptlytest.gpg stretch",
"aptly snapshot create stretch from mirror stretch",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch"
gold_processor = BaseTest.expand_environ
@@ -1160,8 +1189,10 @@ class PublishSnapshot38Test(BaseTest):
"aptly mirror update -keyring=aptlytest.gpg stretch",
"aptly snapshot create stretch from mirror stretch",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch"
gold_processor = BaseTest.expand_environ
sortOutput = True
def check(self):
super(PublishSnapshot38Test, self).check()
@@ -1187,6 +1218,7 @@ class PublishSnapshot39Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap1 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -suite=stable snap1"
gold_processor = BaseTest.expand_environ
@@ -1222,6 +1254,7 @@ class PublishSnapshot40Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap40 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-bz2 snap40"
gold_processor = BaseTest.expand_environ
@@ -1254,6 +1287,7 @@ class PublishSnapshot41Test(BaseTest):
"aptly mirror update -keyring=aptlytest.gpg ps41",
"aptly snapshot create snap41 from mirror ps41",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap41"
gold_processor = BaseTest.expand_environ
@@ -1375,6 +1409,7 @@ class PublishSnapshot42Test(BaseTest):
fixtureCmds = [
"aptly snapshot create snap1 from mirror gnuplot-maverick",
]
sortOutput = True
runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -multi-dist snap1"
gold_processor = BaseTest.expand_environ