system-test: Allow skipping coverage

Enabling coverage near-doubles the incremental build time and adds
overhead to individual tests on the order of **5-10x** or more. It's not
essential to have this for quick local system-test runs, so add an option
to disable it.

Makefile

@@ -15,6 +15,15 @@ export TZ=UTC
 # Unit Tests and some sysmte tests rely on expired certificates, turn back the time
 export TEST_FAKETIME := 2025-01-02 03:04:05
 
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
+else
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
+endif
+
 # export CAPUTRE=1 for regenrating test gold files
 ifeq ($(CAPTURE),1)
 CAPTURE_ARG := --capture
@@ -106,13 +115,13 @@ test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify
 
 system-test: prepare swagger etcd-install  ## Run system tests
 	# build coverage binary
-	go test -v -coverpkg="./..." -c -tags testruncli
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
 	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
 	# Run system tests
-	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST)
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
 
 bench:
 	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -201,7 +210,7 @@ docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirro
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
 		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
 		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
-		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
 		azurite-stop
 
 docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142

system/lib.py

@@ -310,7 +310,9 @@ class BaseTest(object):
 
         if command[0] == "aptly":
             aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test"
-            command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]]
+            command = [str(aptly_testing_bin), *command[1:]]
+            if self.coverage_dir is not None:
+                command.insert(1, f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out")
 
         if self.faketime:
             command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command
@@ -337,7 +339,7 @@ class BaseTest(object):
         if is_aptly_command:
             # remove the last two rows as go tests always print PASS/FAIL and coverage in those
             # two lines. This would otherwise fail the tests as they would not match gold
-            matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output)
+            matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*(?:\ncoverage: .*|$)", raw_output)
             if not matches:
                 raise Exception("no matches found in command output '%s'" % raw_output)
 

system/run.py

@@ -36,7 +36,7 @@ def natural_key(string_):
     return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]
 
 
-def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None):
+def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None, coverage_skip=False):
     """
     Run system test.
     """
@@ -47,7 +47,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     fails = []
     numTests = numFailed = numSkipped = 0
     lastBase = None
-    if not coverage_dir:
+    if not coverage_dir and not coverage_skip:
         coverage_dir = mkdtemp(suffix="aptly-coverage")
 
     failed = False
@@ -213,6 +213,7 @@ if __name__ == "__main__":
     include_long_tests = False
     capture_results = False
     coverage_dir = None
+    coverage_skip = False
     tests = None
     args = sys.argv[1:]
 
@@ -224,6 +225,8 @@ if __name__ == "__main__":
         elif args[0] == "--coverage-dir":
             coverage_dir = args[1]
             args = args[1:]
+        elif args[0] == "--coverage-skip":
+            coverage_skip = True
 
         args = args[1:]
 
@@ -236,4 +239,4 @@ if __name__ == "__main__":
         else:
             filters.append(arg)
 
-    run(include_long_tests, capture_results, tests, filters, coverage_dir)
+    run(include_long_tests, capture_results, tests, filters, coverage_dir, coverage_skip)