diff --git a/Makefile b/Makefile index 3e293292..c8b7628d 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,15 @@ export TZ=UTC # Unit Tests and some sysmte tests rely on expired certificates, turn back the time export TEST_FAKETIME := 2025-01-02 03:04:05 +# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests +ifeq ($(COVERAGE_SKIP),1) +COVERAGE_ARG_BUILD := +COVERAGE_ARG_TEST := --coverage-skip +else +COVERAGE_ARG_BUILD := -coverpkg="./..." +COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR) +endif + # export CAPUTRE=1 for regenrating test gold files ifeq ($(CAPTURE),1) CAPTURE_ARG := --capture @@ -106,13 +115,13 @@ test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify system-test: prepare swagger etcd-install ## Run system tests # build coverage binary - go test -v -coverpkg="./..." -c -tags testruncli + go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli # Download fixture-db, fixture-pool, etcd.db if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) # Run system tests - PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) + PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST) bench: @echo "\e[33m\e[1mRunning benchmark ...\e[0m" @@ -201,7 +210,7 @@ docker-system-test: ## Run system tests in docker container (add TEST=t04_mirro AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ - system-test TEST=$(TEST) CAPTURE=$(CAPTURE) \ + system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \ azurite-stop docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 diff --git a/system/lib.py b/system/lib.py index 77670c86..ff9c890c 100644 --- a/system/lib.py +++ b/system/lib.py @@ -310,7 +310,9 @@ class BaseTest(object): if command[0] == "aptly": aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test" - command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]] + command = [str(aptly_testing_bin), *command[1:]] + if self.coverage_dir is not None: + command.insert(1, f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out") if self.faketime: command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command @@ -337,7 +339,7 @@ class BaseTest(object): if is_aptly_command: # remove the last two rows as go tests always print PASS/FAIL and coverage in those # two lines. This would otherwise fail the tests as they would not match gold - matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output) + matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*(?:\ncoverage: .*|$)", raw_output) if not matches: raise Exception("no matches found in command output '%s'" % raw_output) diff --git a/system/run.py b/system/run.py index 4e73fb2d..2b0de524 100755 --- a/system/run.py +++ b/system/run.py @@ -36,7 +36,7 @@ def natural_key(string_): return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] -def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None): +def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None, coverage_skip=False): """ Run system test. """ @@ -47,7 +47,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non fails = [] numTests = numFailed = numSkipped = 0 lastBase = None - if not coverage_dir: + if not coverage_dir and not coverage_skip: coverage_dir = mkdtemp(suffix="aptly-coverage") failed = False @@ -213,6 +213,7 @@ if __name__ == "__main__": include_long_tests = False capture_results = False coverage_dir = None + coverage_skip = False tests = None args = sys.argv[1:] @@ -224,6 +225,8 @@ if __name__ == "__main__": elif args[0] == "--coverage-dir": coverage_dir = args[1] args = args[1:] + elif args[0] == "--coverage-skip": + coverage_skip = True args = args[1:] @@ -236,4 +239,4 @@ if __name__ == "__main__": else: filters.append(arg) - run(include_long_tests, capture_results, tests, filters, coverage_dir) + run(include_long_tests, capture_results, tests, filters, coverage_dir, coverage_skip)