vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-04-29 21:08:17 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1492 from refi64/docker-test-fixes

Browse Source 
Various fixes for system-test and docker integration
This commit is contained in:
André Roth
2025-10-19 20:33:34 +02:00
committed by GitHub
parent 4526d6d831 33a2f70d07
commit b6d83a4f61
4 changed files with 36 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Makefile
View File

@@ -7,11 +7,23 @@ COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS) GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH) GOARCH=$(shell go env GOHOSTARCH)
export PODMAN_USERNS = keep-id
DOCKER_RUN = docker run --security-opt label=disable -it --user 0:0 --rm -v ${PWD}:/work/src
# Setting TZ for certificates # Setting TZ for certificates
export TZ=UTC export TZ=UTC
# Unit Tests and some sysmte tests rely on expired certificates, turn back the time # Unit Tests and some sysmte tests rely on expired certificates, turn back the time
export TEST_FAKETIME := 2025-01-02 03:04:05 export TEST_FAKETIME := 2025-01-02 03:04:05
# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
ifeq ($(COVERAGE_SKIP),1)
COVERAGE_ARG_BUILD :=
COVERAGE_ARG_TEST := --coverage-skip
else
COVERAGE_ARG_BUILD := -coverpkg="./..."
COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
endif
# export CAPUTRE=1 for regenrating test gold files # export CAPUTRE=1 for regenrating test gold files
ifeq ($(CAPTURE),1) ifeq ($(CAPTURE),1)
CAPTURE_ARG := --capture CAPTURE_ARG := --capture
@@ -103,13 +115,13 @@ test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify
system-test: prepare swagger etcd-install ## Run system tests system-test: prepare swagger etcd-install ## Run system tests
# build coverage binary # build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
# Download fixture-db, fixture-pool, etcd.db # Download fixture-db, fixture-pool, etcd.db
if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests # Run system tests
PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
bench: bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m" @echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -173,16 +185,16 @@ docker-image-no-cache: ## Build aptly-dev docker image (no cache)
@docker build --no-cache -f system/Dockerfile . -t aptly-dev @docker build --no-cache -f system/Dockerfile . -t aptly-dev
docker-build: ## Build aptly in docker container docker-build: ## Build aptly in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper build
docker-shell: ## Run aptly and other commands in docker container docker-shell: ## Run aptly and other commands in docker container
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true @$(DOCKER_RUN) -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
docker-deb: ## Build debian packages in docker container docker-deb: ## Build debian packages in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
docker-unit-test: ## Run unit tests in docker container (add TEST=regex to specify which tests to run) docker-unit-test: ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper \
azurite-start \ azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
@@ -191,27 +203,27 @@ docker-unit-test: ## Run unit tests in docker container (add TEST=regex to spec
azurite-stop azurite-stop
docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper \
azurite-start \ azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
system-test TEST=$(TEST) \ system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
azurite-stop azurite-stop
docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 docker-serve: ## Run development server (auto recompiling) on http://localhost:3142
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true @$(DOCKER_RUN) -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
docker-lint: ## Run golangci-lint in docker container docker-lint: ## Run golangci-lint in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper lint
docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper binaries
docker-man: ## Create man page in docker container docker-man: ## Create man page in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man @$(DOCKER_RUN) aptly-dev /work/src/system/docker-wrapper man
mem.png: mem.dat mem.gp mem.png: mem.dat mem.gp
gnuplot mem.gp gnuplot mem.gp

2
system/docker-wrapper
View File

@@ -15,4 +15,4 @@ else
fi fi
cd /work/src cd /work/src
sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go $cmd sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go GOCACHE=/work/src/.go/cache $cmd

8
system/lib.py
View File

@@ -310,7 +310,9 @@ class BaseTest(object):
if command[0] == "aptly": if command[0] == "aptly":
aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test" aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test"
command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]] command = [str(aptly_testing_bin), *command[1:]]
if self.coverage_dir is not None:
command.insert(1, f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out")
if self.faketime: if self.faketime:
command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command
@@ -337,7 +339,7 @@ class BaseTest(object):
if is_aptly_command: if is_aptly_command:
# remove the last two rows as go tests always print PASS/FAIL and coverage in those # remove the last two rows as go tests always print PASS/FAIL and coverage in those
# two lines. This would otherwise fail the tests as they would not match gold # two lines. This would otherwise fail the tests as they would not match gold
matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output) matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*(?:\ncoverage: .*|$)", raw_output)
if not matches: if not matches:
raise Exception("no matches found in command output '%s'" % raw_output) raise Exception("no matches found in command output '%s'" % raw_output)
@@ -517,7 +519,7 @@ class BaseTest(object):
if gold != output: if gold != output:
diff = "".join(difflib.unified_diff( diff = "".join(difflib.unified_diff(
[l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")])) [l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")]))
raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n") raise Exception(f"content doesn't match:\n{diff}\n\nOutput:\n{orig}\n")
check = check_output check = check_output

9
system/run.py
View File

@@ -36,7 +36,7 @@ def natural_key(string_):
return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]
def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None): def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None, coverage_skip=False):
""" """
Run system test. Run system test.
""" """
@@ -47,7 +47,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
fails = [] fails = []
numTests = numFailed = numSkipped = 0 numTests = numFailed = numSkipped = 0
lastBase = None lastBase = None
if not coverage_dir: if not coverage_dir and not coverage_skip:
coverage_dir = mkdtemp(suffix="aptly-coverage") coverage_dir = mkdtemp(suffix="aptly-coverage")
failed = False failed = False
@@ -213,6 +213,7 @@ if __name__ == "__main__":
include_long_tests = False include_long_tests = False
capture_results = False capture_results = False
coverage_dir = None coverage_dir = None
coverage_skip = False
tests = None tests = None
args = sys.argv[1:] args = sys.argv[1:]
@@ -224,6 +225,8 @@ if __name__ == "__main__":
elif args[0] == "--coverage-dir": elif args[0] == "--coverage-dir":
coverage_dir = args[1] coverage_dir = args[1]
args = args[1:] args = args[1:]
elif args[0] == "--coverage-skip":
coverage_skip = True
args = args[1:] args = args[1:]
@@ -236,4 +239,4 @@ if __name__ == "__main__":
else: else:
filters.append(arg) filters.append(arg)
run(include_long_tests, capture_results, tests, filters, coverage_dir) run(include_long_tests, capture_results, tests, filters, coverage_dir, coverage_skip)