From 4a0bdcbb64054ad627456a735da0892108f4ea81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Tue, 23 Jul 2024 13:05:25 +0200 Subject: [PATCH] improve system tests - log import errors for test modules - log output only on test failure - improve docker system test container - use go 1.19 in docker system tests - download go dependencies in docker container - system tests: color failues output - imrpove test result output - do not install golangci-lint in system tests --- Makefile | 13 +++++++-- system/Dockerfile | 12 ++++---- system/api_lib.py | 23 ++------------- system/lib.py | 3 +- system/run.py | 58 ++++++++++++++++++-------------------- system/t06_publish/list.py | 3 ++ system/t12_api/gpg.py | 1 + system/testout.py | 23 +++++++++++++++ 8 files changed, 75 insertions(+), 61 deletions(-) create mode 100644 system/testout.py diff --git a/Makefile b/Makefile index a67bd7da..5009aacf 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,10 @@ ifeq ($(RUN_LONG_TESTS), yes) endif install: + @echo Building aptly ... go generate - go install -v + @echo go install -v + @out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi system/env: system/requirements.txt ifeq ($(RUN_LONG_TESTS), yes) @@ -57,8 +59,13 @@ ifeq ($(RUN_LONG_TESTS), yes) endif docker-test: install + @echo Building aptly.test ... + @rm -f aptly.test go test -v -coverpkg="./..." -c -tags testruncli - PATH=$(BINPATH)/:$(PATH) APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) + @echo Running python tests ... + export PATH=$(BINPATH)/:$(PATH); \ + export APTLY_VERSION=$(VERSION); \ + $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) test: go test -v ./... -gocheck.v=true -coverprofile=unit.out @@ -93,7 +100,7 @@ version: ## Print aptly version @echo $(VERSION) docker-build-system-tests: ## Build system-test docker image - docker build -f system/Dockerfile --no-cache . -t aptly-system-test + docker build -f system/Dockerfile . -t aptly-system-test docker-unit-tests: ## Run unit tests in docker container docker run -it --rm -v ${PWD}:/app aptly-system-test go test -v ./... -gocheck.v=true diff --git a/system/Dockerfile b/system/Dockerfile index 3b39488e..9988f42a 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -1,10 +1,6 @@ FROM debian:bookworm-slim -RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg && apt-get clean && rm -rf /var/lib/apt/lists/* - -RUN echo deb http://deb.debian.org/debian bookworm-backports main > /etc/apt/sources.list.d/backports.list -RUN apt-get update && \ - apt-get install -y --no-install-recommends apg bzip2 xz-utils ca-certificates golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto && \ +RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ && \ apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly @@ -13,7 +9,13 @@ RUN mkdir app WORKDIR /app RUN mkdir /home/runner +RUN chown aptly /home/runner RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-db.git RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-pool.git +RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db +ADD . /src +RUN chown aptly -R /src +RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy" +RUN rm -rf /src CMD /app/system/run-system-tests diff --git a/system/api_lib.py b/system/api_lib.py index 7e563b30..eef9ad45 100644 --- a/system/api_lib.py +++ b/system/api_lib.py @@ -5,9 +5,9 @@ import random import shutil import string import time -import tempfile from lib import BaseTest +from testout import TestOut try: import requests @@ -19,25 +19,6 @@ TASK_SUCCEEDED = 2 TASK_FAILED = 3 -class AptlyStream: - def __init__(self): - self.tmp_file = tempfile.NamedTemporaryFile(delete=False) - self.read_pos = 0 - - def fileno(self): - return self.tmp_file.fileno() - - def get_contents(self): - self.tmp_file.seek(self.read_pos, 0) - return self.tmp_file.read().decode("utf-8") - - def close(self): - self.tmp_file.close() - - def clear(self): - self.read_pos = self.tmp_file.tell() - - class APITest(BaseTest): """ BaseTest + testing aptly API @@ -62,7 +43,7 @@ class APITest(BaseTest): if APITest.aptly_server is None: super(APITest, self).prepare() - APITest.aptly_out = AptlyStream() + APITest.aptly_out = TestOut() configPath = os.path.join(os.environ["HOME"], self.aptlyConfigFile) APITest.aptly_server = self._start_process(f"aptly api serve -no-lock -config={configPath} -listen={self.base_url}", stdout=APITest.aptly_out, stderr=APITest.aptly_out) diff --git a/system/lib.py b/system/lib.py index 64bb18e9..e3bf521b 100644 --- a/system/lib.py +++ b/system/lib.py @@ -278,6 +278,7 @@ class BaseTest(object): params['url'] = self.webServerUrl command = string.Template(command).substitute(params) + print(f"running command: {command}\n") command = shlex.split(command) if command[0] == "aptly": @@ -294,7 +295,7 @@ class BaseTest(object): proc = self._start_process(command, stdout=subprocess.PIPE) raw_output, _ = proc.communicate() - raw_output = raw_output.decode("utf-8") + raw_output = raw_output.decode("utf-8", errors='replace') returncodes = [proc.returncode] is_aptly_command = False diff --git a/system/run.py b/system/run.py index 71cebe49..dcc3f72b 100755 --- a/system/run.py +++ b/system/run.py @@ -18,6 +18,7 @@ from swift_lib import SwiftTest from azure_lib import AzureTest from api_lib import APITest from fs_endpoint_lib import FileSystemEndpointTest +from testout import TestOut try: from termcolor import colored @@ -34,27 +35,12 @@ def natural_key(string_): return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] -class TestStdout: - def __init__(self): - self.output = [] - - def write(self, text): - self.output.append(text) - - def flush(self): - pass - - def get_output(self): - return ''.join(self.output) - - def truncate(self): - self.output = [] - - def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None): """ Run system test. """ + print(colored("\n Aptly System Tests\n====================\n", color="green", attrs=["bold"])) + if not tests: tests = sorted(glob.glob("t*_*"), key=natural_key) fails = [] @@ -68,18 +54,24 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non orig_stderr = sys.stderr # importlib.import_module(test) - for name in sorted(glob.glob(test + "/*.py"), key=natural_key): - name = os.path.splitext(os.path.basename(name))[0] - if name == "__init__": + for fname in sorted(glob.glob(test + "/*.py"), key=natural_key): + fname = os.path.splitext(os.path.basename(fname))[0] + if fname == "__init__": continue - testout = TestStdout() + testout = TestOut() sys.stdout = testout sys.stderr = testout - testModule = importlib.import_module(test + "." + name) + + try: + testModule = importlib.import_module(test + "." + fname) + except Exception as exc: + orig_stdout.write(f"error importing: {test + '.' + fname}: {exc}\n") + continue for name in sorted(dir(testModule), key=natural_key): - testout.truncate() + testout.clear() + o = getattr(testModule, name) if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and @@ -130,7 +122,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non fails.append((test, t, typ, val, tb, testModule)) orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"])) - orig_stdout.write(testout.get_output()) + orig_stdout.write(testout.get_contents()) traceback.print_exception(typ, val, tb, file=orig_stdout) else: orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"])) @@ -143,19 +135,23 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if lastBase is not None: lastBase.shutdown_class() - print("COVERAGE_RESULTS: %s" % coverage_dir) + print("\nCOVERAGE_RESULTS: %s" % coverage_dir) - print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % ( - numTests, numTests - numFailed, numFailed, numSkipped)) + print(f"TESTS: {numTests} ", + colored(f"SUCCESS: {numTests - numFailed} ", color="green", attrs=["bold"]) if numFailed == 0 else + f"SUCCESS: {numTests - numFailed} ", + colored(f"FAIL: {numFailed} ", color="red", attrs=["bold"]) if numFailed > 0 else "FAIL: 0 ", + colored(f"SKIP: {numSkipped}", color="yellow", attrs=["bold"]) if numSkipped > 0 else "SKIP: 0") + print() if len(fails) > 0: - print("\nFAILURES (%d):" % (len(fails), )) + print(colored("FAILURES (%d):" % (len(fails), ), color="red", attrs=["bold"])) for (test, t, typ, val, tb, testModule) in fails: doc = t.__doc__ or '' - print(" - %s:%s %s" % (test, t.__class__.__name__, - testModule.__name__ + ": " + doc.strip())) - + print(" - %s: %s %s" % (test, colored(t.__class__.__name__, color="yellow", attrs=["bold"]), + testModule.__name__ + ": " + doc.strip())) + print() sys.exit(1) diff --git a/system/t06_publish/list.py b/system/t06_publish/list.py index d5ad7c1f..408edcb2 100644 --- a/system/t06_publish/list.py +++ b/system/t06_publish/list.py @@ -12,6 +12,7 @@ class PublishList2Test(BaseTest): """ publish list: several repos list """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -30,6 +31,7 @@ class PublishList3Test(BaseTest): """ publish list: several repos list, raw """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -54,6 +56,7 @@ class PublishList5Test(BaseTest): """ publish list json: several repos list """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ diff --git a/system/t12_api/gpg.py b/system/t12_api/gpg.py index e05e165a..fb3acd76 100644 --- a/system/t12_api/gpg.py +++ b/system/t12_api/gpg.py @@ -23,6 +23,7 @@ class GPGAPITestAddKey(APITest): """ POST /gpg/key """ + requiresGPG2 = True def check(self): with tempfile.NamedTemporaryFile(suffix=".pub") as keyring: diff --git a/system/testout.py b/system/testout.py new file mode 100644 index 00000000..f7dc134d --- /dev/null +++ b/system/testout.py @@ -0,0 +1,23 @@ +import tempfile + + +class TestOut: + def __init__(self): + self.tmp_file = tempfile.NamedTemporaryFile(delete=False) + self.read_pos = 0 + + def fileno(self): + return self.tmp_file.fileno() + + def write(self, text): + self.tmp_file.write(text.encode()) + + def get_contents(self): + self.tmp_file.seek(self.read_pos, 0) + return self.tmp_file.read().decode("utf-8") + + def close(self): + self.tmp_file.close() + + def clear(self): + self.read_pos = self.tmp_file.tell()