From 22bc2f9d0f14da2d43fcb3d5eaa46ef1c3d8e71f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Mon, 21 Apr 2025 14:13:37 +0200 Subject: [PATCH] system-tests: improve sorted compare sort both aptly output and gold file. output original output for debugging on failure. * Makefile: enable CAPTURE=1 env variable for capturing gold files * docker-system-test: use AWS env vars for S3 tests * fix system tests timing issue with order of gpg logs in publish tests --- Makefile | 12 ++++--- system/lib.py | 29 ++++++++------- system/t06_publish/PublishSnapshot38Test_gold | 1 + system/t06_publish/s3.py | 6 ++++ system/t06_publish/snapshot.py | 35 +++++++++++++++++++ 5 files changed, 66 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 4682f774..fa5d6255 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,10 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) -# Uncomment to update system test gold files -# CAPTURE := "--capture" +# export CAPUTRE=1 for regenrating test gold files +ifeq ($(CAPTURE),1) +CAPTURE_ARG := --capture +endif help: ## Print this help @grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -50,7 +52,7 @@ swagger-install: echo "// @version $(VERSION)" >> docs/swagger.conf azurite-start: - azurite -l /tmp/aptly-azurite & \ + azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \ echo $$! > ~/.azurite.pid azurite-stop: @@ -102,7 +104,7 @@ system-test: prepare swagger etcd-install ## Run system tests if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) # Run system tests - PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) + PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) bench: @echo "\e[33m\e[1mRunning benchmark ...\e[0m" @@ -186,6 +188,8 @@ docker-system-test: ## Run system tests in docker container (add TEST=t04_mirro AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ + AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ + AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ system-test TEST=$(TEST) \ azurite-stop diff --git a/system/lib.py b/system/lib.py index 1287f81d..3f95b102 100644 --- a/system/lib.py +++ b/system/lib.py @@ -289,8 +289,6 @@ class BaseTest(object): def run(self): output = self.run_cmd(self.runCmd, self.expectedCode) - if self.sortOutput: - output = self.sort_lines(output) self.output = self.output_processor(output) def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None): @@ -493,24 +491,29 @@ class BaseTest(object): return a.decode('utf-8') return a - def verify_match(self, a, b, match_prepare=None, ensure_utf8=True): + def verify_match(self, gold, orig, match_prepare=None, ensure_utf8=True): + output = orig if ensure_utf8: - a = self.ensure_utf8(a) - b = self.ensure_utf8(b) + gold = self.ensure_utf8(gold) + output = self.ensure_utf8(output) if match_prepare is not None: - a = match_prepare(a) - b = match_prepare(b) + gold = match_prepare(gold) + output = match_prepare(output) + + # sort if requested + if self.sortOutput: + gold = self.sort_lines(gold) + output = self.sort_lines(output) # strip trailing whitespace and newlines - a = a.strip() - b = b.strip() + gold = gold.strip() + output = output.strip() - if a != b: + if gold != output: diff = "".join(difflib.unified_diff( - [l + "\n" for l in a.split("\n")], [l + "\n" for l in b.split("\n")])) - - raise Exception("content doesn't match:\n" + diff + "\n") + [l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")])) + raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n") check = check_output diff --git a/system/t06_publish/PublishSnapshot38Test_gold b/system/t06_publish/PublishSnapshot38Test_gold index 8b9bb3cb..64d9fe57 100644 --- a/system/t06_publish/PublishSnapshot38Test_gold +++ b/system/t06_publish/PublishSnapshot38Test_gold @@ -12,3 +12,4 @@ Now you can add following line to apt sources: Don't forget to add your GPG key to apt with apt-key. You can also use `aptly serve` to publish your repositories over HTTP quickly. + diff --git a/system/t06_publish/s3.py b/system/t06_publish/s3.py index c818023a..f57adb06 100644 --- a/system/t06_publish/s3.py +++ b/system/t06_publish/s3.py @@ -14,6 +14,7 @@ class S3Publish1Test(S3Test): "aptly repo add local-repo ${files}", "aptly repo remove local-repo libboost-program-options-dev_1.62.0.1_i386", ] + sortOutput = True runCmd = "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:" def check(self): @@ -53,6 +54,7 @@ class S3Publish2Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:", "aptly repo remove local-repo pyspi" ] + sortOutput = True runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:" def check(self): @@ -93,6 +95,7 @@ class S3Publish3Test(S3Test): "aptly snapshot pull -no-deps -architectures=i386,amd64 snap2 snap1 snap3 gnuplot-x11", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1 s3:test1:", ] + sortOutput = True runCmd = "aptly publish switch -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1: snap3" def check(self): @@ -129,6 +132,7 @@ class S3Publish4Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=xyz local-repo s3:test1:", "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:prefix", ] + sortOutput = True runCmd = "aptly publish list" @@ -144,6 +148,7 @@ class S3Publish5Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq1 local1 s3:test1:", "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=sq2 local2 s3:test1:", ] + sortOutput = True runCmd = "aptly publish drop sq2 s3:test1:" def check(self): @@ -172,6 +177,7 @@ class S3Publish6Test(S3Test): "aptly publish repo -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec local-repo s3:test1:", "aptly repo remove local-repo pyspi" ] + sortOutput = True runCmd = "aptly publish update -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec maverick s3:test1:" def check(self): diff --git a/system/t06_publish/snapshot.py b/system/t06_publish/snapshot.py index 2ba245b4..fb877bb7 100644 --- a/system/t06_publish/snapshot.py +++ b/system/t06_publish/snapshot.py @@ -22,6 +22,7 @@ class PublishSnapshot1Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap1" gold_processor = BaseTest.expand_environ @@ -131,6 +132,7 @@ class PublishSnapshot2Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap2 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" gold_processor = BaseTest.expand_environ @@ -168,6 +170,7 @@ class PublishSnapshot3Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap3 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -component=contrib snap3" gold_processor = BaseTest.expand_environ @@ -208,6 +211,7 @@ class PublishSnapshot4Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap4 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly -architectures=i386 publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap4" gold_processor = BaseTest.expand_environ @@ -247,6 +251,7 @@ class PublishSnapshot5Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap5 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -acquire-by-hash -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap5 ppa/smira" gold_processor = BaseTest.expand_environ @@ -294,6 +299,7 @@ class PublishSnapshot6Test(BaseTest): "aptly snapshot create snap2 from mirror wheezy-main", "aptly snapshot merge snap6 snap2 snap" ] + sortOutput = True runCmd = "aptly publish snapshot snap6" expectedCode = 1 @@ -308,6 +314,7 @@ class PublishSnapshot7Test(BaseTest): "aptly snapshot create snap7 from mirror gnuplot-maverick", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap7", ] + sortOutput = True runCmd = "aptly publish snapshot snap7" expectedCode = 1 @@ -322,6 +329,7 @@ class PublishSnapshot8Test(BaseTest): "aptly snapshot create snap8 from mirror gnuplot-maverick", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap8 ./ppa", ] + sortOutput = True runCmd = "aptly publish snapshot snap8 ppa" expectedCode = 1 @@ -334,6 +342,7 @@ class PublishSnapshot9Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap9 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap9 ppa/dists/la" expectedCode = 1 @@ -346,6 +355,7 @@ class PublishSnapshot10Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap10 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap10 ppa/pool/la" expectedCode = 1 @@ -358,6 +368,7 @@ class PublishSnapshot11Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap11 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot snap11 ../la" expectedCode = 1 @@ -367,6 +378,7 @@ class PublishSnapshot12Test(BaseTest): publish snapshot: no snapshot """ fixtureDB = True + sortOutput = True runCmd = "aptly publish snapshot snap12" expectedCode = 1 @@ -465,6 +477,7 @@ class PublishSnapshot16Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap16 from mirror gnuplot-maverick-src", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap16" gold_processor = BaseTest.expand_environ @@ -524,6 +537,7 @@ class PublishSnapshot17Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap17 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap17" gold_processor = BaseTest.expand_environ @@ -578,6 +592,7 @@ class PublishSnapshot18Test(BaseTest): "aptly repo add repo1 ${files}", "aptly snapshot create snap1 from repo repo1", ] + sortOutput = True runCmd = "aptly publish snapshot snap1" expectedCode = 1 @@ -682,6 +697,7 @@ class PublishSnapshot24Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap24 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze -origin=aptly24 -notautomatic=yes -butautomaticupgrades=yes snap24" gold_processor = BaseTest.expand_environ @@ -701,6 +717,7 @@ class PublishSnapshot25Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap25 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -architectures=amd64 --distribution=maverick -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap25" gold_processor = BaseTest.expand_environ @@ -736,6 +753,7 @@ class PublishSnapshot26Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap26.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=main,contrib snap26.1 snap26.2" gold_processor = BaseTest.expand_environ @@ -864,6 +882,7 @@ class PublishSnapshot27Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap27.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -component=, snap27.1 snap27.2" gold_processor = BaseTest.expand_environ @@ -879,6 +898,7 @@ class PublishSnapshot28Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap28.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -component=, snap28.1 snap28.2" expectedCode = 1 @@ -891,6 +911,7 @@ class PublishSnapshot29Test(BaseTest): "aptly snapshot create snap29.1 empty", "aptly snapshot create snap29.2 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=b,b snap29.1 snap29.2" expectedCode = 1 @@ -906,6 +927,7 @@ class PublishSnapshot30Test(BaseTest): "aptly repo add local-repo ${files}", "aptly snapshot create snap30.2 from repo local-repo", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap30.1 snap30.2" expectedCode = 1 @@ -917,6 +939,7 @@ class PublishSnapshot31Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap31.1 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap31.1 snap31.2" expectedCode = 1 @@ -928,6 +951,7 @@ class PublishSnapshot32Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap32.1 empty", ] + sortOutput = True runCmd = "aptly publish snapshot -component=main,contrib snap32.1" expectedCode = 2 @@ -948,6 +972,7 @@ class PublishSnapshot33Test(BaseTest): "aptly snapshot create snap2 from repo local-repo2", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" expectedCode = 1 gold_processor = BaseTest.expand_environ @@ -966,6 +991,7 @@ class PublishSnapshot34Test(BaseTest): "aptly snapshot create snap2 from repo local-repo2", "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=maverick snap1", ] + sortOutput = True runCmd = "aptly publish snapshot -force-overwrite -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -distribution=squeeze snap2" gold_processor = BaseTest.expand_environ @@ -987,6 +1013,7 @@ class PublishSnapshot35Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ @@ -1118,6 +1145,7 @@ class PublishSnapshot36Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap36 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-contents snap36" gold_processor = BaseTest.expand_environ @@ -1145,6 +1173,7 @@ class PublishSnapshot37Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ @@ -1160,8 +1189,10 @@ class PublishSnapshot38Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg stretch", "aptly snapshot create stretch from mirror stretch", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec stretch" gold_processor = BaseTest.expand_environ + sortOutput = True def check(self): super(PublishSnapshot38Test, self).check() @@ -1187,6 +1218,7 @@ class PublishSnapshot39Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -suite=stable snap1" gold_processor = BaseTest.expand_environ @@ -1222,6 +1254,7 @@ class PublishSnapshot40Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap40 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -skip-bz2 snap40" gold_processor = BaseTest.expand_environ @@ -1254,6 +1287,7 @@ class PublishSnapshot41Test(BaseTest): "aptly mirror update -keyring=aptlytest.gpg ps41", "aptly snapshot create snap41 from mirror ps41", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec snap41" gold_processor = BaseTest.expand_environ @@ -1375,6 +1409,7 @@ class PublishSnapshot42Test(BaseTest): fixtureCmds = [ "aptly snapshot create snap1 from mirror gnuplot-maverick", ] + sortOutput = True runCmd = "aptly publish snapshot -keyring=${files}/aptly.pub -secret-keyring=${files}/aptly.sec -multi-dist snap1" gold_processor = BaseTest.expand_environ