From 98b1ed07d1a77796928a34a8bb5e876ba2a81ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Fri, 2 Aug 2024 23:17:29 +0200 Subject: [PATCH] docker: improve dev env - abort docker scripts on error - generate version in system tests - build debian packages in docker - add make clean target - fix lint --- .dockerignore | 8 +++++-- CONTRIBUTING.md | 4 ++-- Makefile | 20 ++++++++++-------- system/Dockerfile | 18 ++++++++-------- system/build-deb | 16 ++++++++++++++ system/lib.py | 2 +- system/run-aptly-cmd | 21 +++++++------------ system/run-golangci-lint | 15 ++----------- system/run-system-tests | 4 +--- system/run-unit-tests | 5 +---- system/t04_mirror/CreateMirror18Test_gold | 1 - system/t04_mirror/CreateMirror30Test_gold | 2 +- system/t04_mirror/create.py | 3 ++- system/t06_publish/PublishSnapshot41Test_gold | 2 +- 14 files changed, 60 insertions(+), 61 deletions(-) create mode 100755 system/build-deb diff --git a/.dockerignore b/.dockerignore index cf1420f4..41db8dbc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,6 @@ -.go -.git +.go/ +.git/ +obj-x86_64-linux-gnu/ +unit.out +aptly.test +build/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7744e7f5..edebdf22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,9 +103,9 @@ Install the following on your development machine: #### Create docker container -To build the development docker container, run: +To build the development docker image, run: ``` -make docker-build-aptly-dev +make docker-image ``` #### Build aptly diff --git a/Makefile b/Makefile index 3a3f3b0b..cbb49ed1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,5 @@ -GOVERSION=$(shell go version | awk '{print $$3;}') GOPATH=$(shell go env GOPATH) VERSION=$(shell make version) -PACKAGES=context database deb files gpg http query swift s3 utils PYTHON?=python3 TESTS?= BINPATH?=$(GOPATH)/bin @@ -60,9 +58,10 @@ ifeq ($(RUN_LONG_TESTS), yes) PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) endif -docker-test: install ## Run system tests +docker-test: ## Run system tests @echo Building aptly.test ... @rm -f aptly.test + go generate go test -v -coverpkg="./..." -c -tags testruncli @echo Running python tests ... @test -e aws.creds && . ./aws.creds; \ @@ -119,14 +118,17 @@ build: ## Build aptly go generate go build -o build/aptly -docker-build-aptly-dev: ## Build aptly-dev docker image +docker-image: ## Build aptly-dev docker image @docker build -f system/Dockerfile . -t aptly-dev docker-build: ## Build aptly in docker container - @docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-aptly-cmd make build + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build docker-aptly: ## Build and run aptly commands in docker container - @docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-aptly-cmd + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd + +docker-deb: ## Build debian packages in docker container + @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/build-deb docker-unit-tests: ## Run unit tests in docker container @docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-unit-tests @@ -137,11 +139,11 @@ docker-system-tests: ## Run system tests in docker container (add TEST=t04_mirr docker-lint: ## Run golangci-lint in docker container @docker run -it --rm -v ${PWD}:/app -e GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) aptly-dev /app/system/run-golangci-lint -flake8: ## run flak8 on system tests +flake8: ## run flake8 on system tests flake8 system clean: ## remove local build and module cache - test -f .go/ && chmod u+w -R .go/; rm -rf .go/ - rm -rf build/ + test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ + rm -rf build/ docs/ obj-x86_64-linux-gnu/ .PHONY: help man modules version release goxc docker-build-aptly-dev docker-system-tests docker-unit-tests docker-lint docker-build build docker-aptly clean diff --git a/system/Dockerfile b/system/Dockerfile index 11a21ba2..8b2cbbe4 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -1,21 +1,21 @@ FROM debian:bookworm-slim -RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel graphviz && \ +RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf bash-completion && \ apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly -RUN mkdir app -WORKDIR /app +RUN mkdir /work +WORKDIR /work +RUN chown aptly /work -RUN mkdir /home/runner -RUN chown aptly /home/runner -RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-db.git -RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-pool.git -RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db +RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-db.git +RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-pool.git +RUN cd /var/lib/aptly; curl -O http://repo.aptly.info/system-tests/etcd.db +RUN echo "aptly ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/aptly ADD . /src RUN chown aptly -R /src -RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy" +RUN cd /src; su aptly -c "go mod download" RUN /src/system/t13_etcd/install-etcd.sh RUN rm -rf /src diff --git a/system/build-deb b/system/build-deb new file mode 100755 index 00000000..cd06189c --- /dev/null +++ b/system/build-deb @@ -0,0 +1,16 @@ +#!/bin/sh -e + +usermod -u `stat -c %u /work/src` aptly >/dev/null +chown -R `stat -c %u /work/src` /var/lib/aptly + + +su aptly -c 'set -e; cd /work/src; +GOPATH=$PWD/.go go generate -v +DEBEMAIL="CI " dch -v `make version` "CI build" +dpkg-buildpackage -us -uc -b -d --host-arch=amd64 +dpkg-buildpackage -us -uc -b -d --host-arch=i386 +dpkg-buildpackage -us -uc -b -d --host-arch=arm64 +dpkg-buildpackage -us -uc -b -d --host-arch=armhf +mkdir -p build && mv ../*.deb build/ +cd build && ls -l *.deb +' diff --git a/system/lib.py b/system/lib.py index a2e14d0e..9206c06a 100644 --- a/system/lib.py +++ b/system/lib.py @@ -275,7 +275,7 @@ class BaseTest(object): if self.fixtureGpg: self.run_cmd([self.gpgFinder.gpg, "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] + [os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", key) for key in self.fixtureGpgKeys]) - self.run_cmd(["chmod", "400", "/home/runner/.gnupg/aptlytest.gpg"]) + self.run_cmd(["chmod", "400", os.path.join(os.environ["HOME"], ".gnupg/aptlytest.gpg")]) if hasattr(self, "fixtureCmds"): for cmd in self.fixtureCmds: diff --git a/system/run-aptly-cmd b/system/run-aptly-cmd index ec76b75a..b3d0ad1f 100755 --- a/system/run-aptly-cmd +++ b/system/run-aptly-cmd @@ -1,17 +1,10 @@ #!/bin/sh -# cleanup -rm -rf /app/tmp -rm -rf /tmp/aptly* - -usermod -u `stat -c %u /app` aptly >/dev/null -chown -R `stat -c %u /app` /var/lib/aptly - -# use same /home/runner dir as in github workflow -chown -R `stat -c %u /app` /home/runner - cmd=$@ -if [ -z "$cmd" ]; then - cmd="bash" -fi -su aptly -c "cd /app; export HOME=/home/runner; export PATH=\$PATH:/app/build; $cmd" +test -z "$cmd" && cmd="bash" + +usermod -u `stat -c %u /work/src` aptly >/dev/null +chown -R `stat -c %u /work/src` /var/lib/aptly + +cd /work/src +exec sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go $cmd diff --git a/system/run-golangci-lint b/system/run-golangci-lint index e59320cd..14921121 100755 --- a/system/run-golangci-lint +++ b/system/run-golangci-lint @@ -1,17 +1,6 @@ -#!/bin/sh - -# cleanup -rm -rf /app/tmp -rm -rf /tmp/aptly* +#!/bin/sh -e usermod -u `stat -c %u /app` aptly >/dev/null chown -R `stat -c %u /app` /var/lib/aptly -# use same /home/runner dir as in github workflow -chown -R `stat -c %u /app` /home/runner - -cmd=$@ -if [ -z "$cmd" ]; then - cmd="bash" -fi -su aptly -c "cd /app; GOPATH=/app/.go go install github.com/golangci/golangci-lint/cmd/golangci-lint@$GOLANGCI_LINT_VERSION; PATH=\$PATH:/app/.go/bin golangci-lint run" +su aptly -c "set -e; cd /app; export GOPATH=/app/.go; go install github.com/golangci/golangci-lint/cmd/golangci-lint@$GOLANGCI_LINT_VERSION; PATH=\$PATH:/app/.go/bin golangci-lint run" diff --git a/system/run-system-tests b/system/run-system-tests index add0f186..64a0a709 100755 --- a/system/run-system-tests +++ b/system/run-system-tests @@ -7,6 +7,4 @@ rm -rf /tmp/aptly* usermod -u `stat -c %u /app` aptly >/dev/null chown -R `stat -c %u /app` /var/lib/aptly -# use same /home/runner dir as in github workflow -chown -R `stat -c %u /app` /home/runner -su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make docker-test TEST=$@" +su - aptly -c "cd /app; export GOPATH=/app/.go; go mod tidy; make docker-test TEST=$@" diff --git a/system/run-unit-tests b/system/run-unit-tests index 80a4022c..34b98bc4 100755 --- a/system/run-unit-tests +++ b/system/run-unit-tests @@ -5,10 +5,7 @@ rm -rf /app/tmp rm -rf /tmp/aptly* mkdir -p /srv - usermod -u `stat -c %u /app` aptly >/dev/null chown -R `stat -c %u /app` /var/lib/aptly /srv -# use same /home/runner dir as in github workflow -chown -R `stat -c %u /app` /home/runner -su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make test" +su - aptly -c "cd /app; export export GOPATH=/app/.go; go mod tidy; make test" diff --git a/system/t04_mirror/CreateMirror18Test_gold b/system/t04_mirror/CreateMirror18Test_gold index 0444a91e..d63b8745 100644 --- a/system/t04_mirror/CreateMirror18Test_gold +++ b/system/t04_mirror/CreateMirror18Test_gold @@ -1,5 +1,4 @@ Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease -gpgv: can't allocate lock for '/home/runner/.gnupg/ppa.gpg' gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC gpgv: using RSA key 5BFCD481D86D5824470E469F9000B1C3A01F726C gpgv: Good signature from "Launchpad PPA for Anton Gladky" diff --git a/system/t04_mirror/CreateMirror30Test_gold b/system/t04_mirror/CreateMirror30Test_gold index 953e0e04..ae944b8e 100644 --- a/system/t04_mirror/CreateMirror30Test_gold +++ b/system/t04_mirror/CreateMirror30Test_gold @@ -1,4 +1,4 @@ -opengpg: failure opening keyring '/home/runner/.gnupg/aptlytest.gpg': open /home/runner/.gnupg/aptlytest.gpg: no such file or directory +opengpg: failure opening keyring '${HOME}/.gnupg/aptlytest.gpg': open ${HOME}/.gnupg/aptlytest.gpg: no such file or directory Looks like your keyring with trusted keys is empty. You might consider importing some keys. Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease diff --git a/system/t04_mirror/create.py b/system/t04_mirror/create.py index d02350e4..ef3d80f6 100644 --- a/system/t04_mirror/create.py +++ b/system/t04_mirror/create.py @@ -223,7 +223,8 @@ class CreateMirror18Test(BaseTest): } fixtureCmds = [ - "gpg --no-default-keyring --keyring=ppa.gpg --keyserver=hkp://keyserver.ubuntu.com:80 --recv-keys 5BFCD481D86D5824470E469F9000B1C3A01F726C 02219381E9161C78A46CB2BFA5279A973B1F56C0" + "gpg --no-default-keyring --keyring=ppa.gpg --keyserver=hkp://keyserver.ubuntu.com:80 --recv-keys 5BFCD481D86D5824470E469F9000B1C3A01F726C 02219381E9161C78A46CB2BFA5279A973B1F56C0", + f"chmod 400 {os.path.join(os.environ['HOME'], '.gnupg/ppa.gpg')}" ] runCmd = "aptly mirror create -keyring=ppa.gpg mirror18 ppa:gladky-anton/gnuplot" diff --git a/system/t06_publish/PublishSnapshot41Test_gold b/system/t06_publish/PublishSnapshot41Test_gold index 46e6a5e9..06f16c33 100644 --- a/system/t06_publish/PublishSnapshot41Test_gold +++ b/system/t06_publish/PublishSnapshot41Test_gold @@ -5,7 +5,7 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted: Clearsigning file 'Release' with gpg, please enter your passphrase when prompted: Snapshot snap41 has been successfully published. -Please setup your webserver to serve directory '/home/runner/.aptly/public' with autoindexing. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. Now you can add following line to apt sources: deb http://your-server/ buster/updates main deb-src http://your-server/ buster/updates main