improve system tests

- log import errors for test modules
- log output only on test failure
- improve docker system test container
- use go 1.19 in docker system tests
- download go dependencies in docker container
- system tests: color failues output
- imrpove test result output
- do not install golangci-lint in system tests

Makefile

@@ -37,8 +37,10 @@ ifeq ($(RUN_LONG_TESTS), yes)
 endif
 
 install:
+	@echo Building aptly ...
 	go generate
-	go install -v
+	@echo go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
 system/env: system/requirements.txt
 ifeq ($(RUN_LONG_TESTS), yes)
@@ -57,8 +59,13 @@ ifeq ($(RUN_LONG_TESTS), yes)
 endif
 
 docker-test: install
+	@echo Building aptly.test ...
+	@rm -f aptly.test
 	go test -v -coverpkg="./..." -c -tags testruncli
-	PATH=$(BINPATH)/:$(PATH) APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
+	@echo Running python tests ...
+	export PATH=$(BINPATH)/:$(PATH); \
+	export APTLY_VERSION=$(VERSION); \
+	$(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
 test:
 	go test -v ./... -gocheck.v=true -coverprofile=unit.out
@@ -93,7 +100,7 @@ version:  ## Print aptly version
 	@echo $(VERSION)
 
 docker-build-system-tests:  ## Build system-test docker image
-	docker build -f system/Dockerfile --no-cache . -t aptly-system-test
+	docker build -f system/Dockerfile . -t aptly-system-test
 
 docker-unit-tests:  ## Run unit tests in docker container
 	docker run -it --rm -v ${PWD}:/app aptly-system-test go test -v ./... -gocheck.v=true

system/Dockerfile

@@ -1,10 +1,6 @@
 FROM debian:bookworm-slim
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg && apt-get clean && rm -rf /var/lib/apt/lists/*
+RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ && \
-
-RUN echo deb http://deb.debian.org/debian bookworm-backports main > /etc/apt/sources.list.d/backports.list
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends apg bzip2 xz-utils ca-certificates golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
@@ -13,7 +9,13 @@ RUN mkdir app
 WORKDIR /app
 
 RUN mkdir /home/runner
+RUN chown aptly /home/runner
 RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-db.git
 RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-pool.git
+RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db
+ADD . /src
+RUN chown aptly -R /src
+RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy"
+RUN rm -rf /src
 
 CMD /app/system/run-system-tests

system/api_lib.py

@@ -5,9 +5,9 @@ import random
 import shutil
 import string
 import time
-import tempfile
 
 from lib import BaseTest
+from testout import TestOut
 
 try:
     import requests
@@ -19,25 +19,6 @@ TASK_SUCCEEDED = 2
 TASK_FAILED = 3
 
 
-class AptlyStream:
-    def __init__(self):
-        self.tmp_file = tempfile.NamedTemporaryFile(delete=False)
-        self.read_pos = 0
-
-    def fileno(self):
-        return self.tmp_file.fileno()
-
-    def get_contents(self):
-        self.tmp_file.seek(self.read_pos, 0)
-        return self.tmp_file.read().decode("utf-8")
-
-    def close(self):
-        self.tmp_file.close()
-
-    def clear(self):
-        self.read_pos = self.tmp_file.tell()
-
-
 class APITest(BaseTest):
     """
     BaseTest + testing aptly API
@@ -62,7 +43,7 @@ class APITest(BaseTest):
         if APITest.aptly_server is None:
             super(APITest, self).prepare()
 
-            APITest.aptly_out = AptlyStream()
+            APITest.aptly_out = TestOut()
 
             configPath = os.path.join(os.environ["HOME"], self.aptlyConfigFile)
             APITest.aptly_server = self._start_process(f"aptly api serve -no-lock -config={configPath} -listen={self.base_url}", stdout=APITest.aptly_out, stderr=APITest.aptly_out)

system/lib.py

@@ -278,6 +278,7 @@ class BaseTest(object):
                 params['url'] = self.webServerUrl
 
             command = string.Template(command).substitute(params)
+            print(f"running command: {command}\n")
             command = shlex.split(command)
 
         if command[0] == "aptly":
@@ -294,7 +295,7 @@ class BaseTest(object):
             proc = self._start_process(command, stdout=subprocess.PIPE)
             raw_output, _ = proc.communicate()
 
-            raw_output = raw_output.decode("utf-8")
+            raw_output = raw_output.decode("utf-8", errors='replace')
 
             returncodes = [proc.returncode]
             is_aptly_command = False

system/run.py

@@ -18,6 +18,7 @@ from swift_lib import SwiftTest
 from azure_lib import AzureTest
 from api_lib import APITest
 from fs_endpoint_lib import FileSystemEndpointTest
+from testout import TestOut
 
 try:
     from termcolor import colored
@@ -34,27 +35,12 @@ def natural_key(string_):
     return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]
 
 
-class TestStdout:
-    def __init__(self):
-        self.output = []
-
-    def write(self, text):
-        self.output.append(text)
-
-    def flush(self):
-        pass
-
-    def get_output(self):
-        return ''.join(self.output)
-
-    def truncate(self):
-        self.output = []
-
-
 def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None):
     """
     Run system test.
     """
+    print(colored("\n Aptly System Tests\n====================\n", color="green", attrs=["bold"]))
+
     if not tests:
         tests = sorted(glob.glob("t*_*"), key=natural_key)
     fails = []
@@ -68,18 +54,24 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
         orig_stderr = sys.stderr
 
         # importlib.import_module(test)
-        for name in sorted(glob.glob(test + "/*.py"), key=natural_key):
+        for fname in sorted(glob.glob(test + "/*.py"), key=natural_key):
-            name = os.path.splitext(os.path.basename(name))[0]
+            fname = os.path.splitext(os.path.basename(fname))[0]
-            if name == "__init__":
+            if fname == "__init__":
                 continue
 
-            testout = TestStdout()
+            testout = TestOut()
             sys.stdout = testout
             sys.stderr = testout
-            testModule = importlib.import_module(test + "." + name)
+
+            try:
+                testModule = importlib.import_module(test + "." + fname)
+            except Exception as exc:
+                orig_stdout.write(f"error importing: {test + '.' + fname}: {exc}\n")
+                continue
 
             for name in sorted(dir(testModule), key=natural_key):
-                testout.truncate()
+                testout.clear()
+
                 o = getattr(testModule, name)
 
                 if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
@@ -130,7 +122,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                     fails.append((test, t, typ, val, tb, testModule))
                     orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"]))
 
-                    orig_stdout.write(testout.get_output())
+                    orig_stdout.write(testout.get_contents())
                     traceback.print_exception(typ, val, tb, file=orig_stdout)
                 else:
                     orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"]))
@@ -143,19 +135,23 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     if lastBase is not None:
         lastBase.shutdown_class()
 
-    print("COVERAGE_RESULTS: %s" % coverage_dir)
+    print("\nCOVERAGE_RESULTS: %s" % coverage_dir)
 
-    print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % (
+    print(f"TESTS: {numTests}    ",
-        numTests, numTests - numFailed, numFailed, numSkipped))
+          colored(f"SUCCESS: {numTests - numFailed}    ", color="green", attrs=["bold"]) if numFailed == 0 else
+          f"SUCCESS: {numTests - numFailed}    ",
+          colored(f"FAIL: {numFailed}    ", color="red", attrs=["bold"]) if numFailed > 0 else "FAIL: 0    ",
+          colored(f"SKIP: {numSkipped}", color="yellow", attrs=["bold"]) if numSkipped > 0 else "SKIP: 0")
+    print()
 
     if len(fails) > 0:
-        print("\nFAILURES (%d):" % (len(fails), ))
+        print(colored("FAILURES (%d):" % (len(fails), ), color="red", attrs=["bold"]))
 
         for (test, t, typ, val, tb, testModule) in fails:
             doc = t.__doc__ or ''
-            print(" - %s:%s %s" % (test, t.__class__.__name__,
+            print(" - %s: %s %s" % (test, colored(t.__class__.__name__, color="yellow", attrs=["bold"]),
-                                   testModule.__name__ + ": " + doc.strip()))
+                                    testModule.__name__ + ": " + doc.strip()))
-
+        print()
         sys.exit(1)
 
 

system/t06_publish/list.py

@@ -12,6 +12,7 @@ class PublishList2Test(BaseTest):
     """
     publish list: several repos list
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -30,6 +31,7 @@ class PublishList3Test(BaseTest):
     """
     publish list: several repos list, raw
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -54,6 +56,7 @@ class PublishList5Test(BaseTest):
     """
     publish list json: several repos list
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [

system/t12_api/gpg.py

@@ -23,6 +23,7 @@ class GPGAPITestAddKey(APITest):
     """
     POST /gpg/key
     """
+    requiresGPG2 = True
 
     def check(self):
         with tempfile.NamedTemporaryFile(suffix=".pub") as keyring:

system/testout.py

@@ -0,0 +1,23 @@
+import tempfile
+
+
+class TestOut:
+    def __init__(self):
+        self.tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        self.read_pos = 0
+
+    def fileno(self):
+        return self.tmp_file.fileno()
+
+    def write(self, text):
+        self.tmp_file.write(text.encode())
+
+    def get_contents(self):
+        self.tmp_file.seek(self.read_pos, 0)
+        return self.tmp_file.read().decode("utf-8")
+
+    def close(self):
+        self.tmp_file.close()
+
+    def clear(self):
+        self.read_pos = self.tmp_file.tell()