docker: improve dev env

- abort docker scripts on error
- generate version in system tests
- build debian packages in docker
- add make clean target
- fix lint

.dockerignore

@@ -1,2 +1,6 @@
-.go
+.go/
-.git
+.git/
+obj-x86_64-linux-gnu/
+unit.out
+aptly.test
+build/

CONTRIBUTING.md

@@ -103,9 +103,9 @@ Install the following on your development machine:
 
 #### Create docker container
 
-To build the development docker container, run:
+To build the development docker image, run:
 ```
-make docker-build-aptly-dev
+make docker-image
 ```
 
 #### Build aptly

Makefile

@@ -1,7 +1,5 @@
-GOVERSION=$(shell go version | awk '{print $$3;}')
 GOPATH=$(shell go env GOPATH)
 VERSION=$(shell make version)
-PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python3
 TESTS?=
 BINPATH?=$(GOPATH)/bin
@@ -60,9 +58,10 @@ ifeq ($(RUN_LONG_TESTS), yes)
 	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
 endif
 
-docker-test: install  ## Run system tests
+docker-test: ## Run system tests
 	@echo Building aptly.test ...
 	@rm -f aptly.test
+	go generate
 	go test -v -coverpkg="./..." -c -tags testruncli
 	@echo Running python tests ...
 	@test -e aws.creds && . ./aws.creds; \
@@ -119,14 +118,17 @@ build:  ## Build aptly
 	go generate
 	go build -o build/aptly
 
-docker-build-aptly-dev:  ## Build aptly-dev docker image
+docker-image:  ## Build aptly-dev docker image
 	@docker build -f system/Dockerfile . -t aptly-dev
 
 docker-build:  ## Build aptly in docker container
-	@docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-aptly-cmd make build
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build
 
 docker-aptly:  ## Build and run aptly commands in docker container
-	@docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-aptly-cmd
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd
+
+docker-deb:  ## Build debian packages in docker container
+	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/build-deb
 
 docker-unit-tests:  ## Run unit tests in docker container
 	@docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-unit-tests
@@ -137,11 +139,11 @@ docker-system-tests:  ## Run system tests in docker container (add TEST=t04_mirr
 docker-lint:  ## Run golangci-lint in docker container
 	@docker run -it --rm -v ${PWD}:/app -e GOLANGCI_LINT_VERSION=$(GOLANGCI_LINT_VERSION) aptly-dev /app/system/run-golangci-lint
 
-flake8:  ## run flak8 on system tests
+flake8:  ## run flake8 on system tests
 	flake8 system
 
 clean:  ## remove local build and module cache
-	test -f .go/ && chmod u+w -R .go/; rm -rf .go/
+	test -d .go/ && chmod u+w -R .go/ && rm -rf .go/
-	rm -rf build/
+	rm -rf build/ docs/ obj-x86_64-linux-gnu/
 
 .PHONY: help man modules version release goxc docker-build-aptly-dev docker-system-tests docker-unit-tests docker-lint docker-build build docker-aptly clean

system/Dockerfile

@@ -1,21 +1,21 @@
 FROM debian:bookworm-slim
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel graphviz && \
+RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf bash-completion && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
 
-RUN mkdir app
+RUN mkdir /work
-WORKDIR /app
+WORKDIR /work
+RUN chown aptly /work
 
-RUN mkdir /home/runner
+RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-db.git
-RUN chown aptly /home/runner
+RUN cd /var/lib/aptly; git clone https://github.com/aptly-dev/aptly-fixture-pool.git
-RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-db.git
+RUN cd /var/lib/aptly; curl -O http://repo.aptly.info/system-tests/etcd.db
-RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-pool.git
+RUN echo "aptly ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/aptly
-RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db
 ADD . /src
 RUN chown aptly -R /src
-RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy"
+RUN cd /src; su aptly -c "go mod download"
 RUN /src/system/t13_etcd/install-etcd.sh
 RUN rm -rf /src
 

system/build-deb

@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+usermod -u `stat -c %u /work/src` aptly >/dev/null
+chown -R `stat -c %u /work/src` /var/lib/aptly
+
+
+su aptly -c 'set -e; cd /work/src;
+GOPATH=$PWD/.go go generate -v
+DEBEMAIL="CI <runner@github>" dch -v `make version` "CI build"
+dpkg-buildpackage -us -uc -b -d --host-arch=amd64
+dpkg-buildpackage -us -uc -b -d --host-arch=i386
+dpkg-buildpackage -us -uc -b -d --host-arch=arm64
+dpkg-buildpackage -us -uc -b -d --host-arch=armhf
+mkdir -p build && mv ../*.deb build/
+cd build && ls -l *.deb
+'

system/lib.py

@@ -275,7 +275,7 @@ class BaseTest(object):
         if self.fixtureGpg:
             self.run_cmd([self.gpgFinder.gpg, "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] +
                          [os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files", key) for key in self.fixtureGpgKeys])
-            self.run_cmd(["chmod", "400", "/home/runner/.gnupg/aptlytest.gpg"])
+            self.run_cmd(["chmod", "400", os.path.join(os.environ["HOME"], ".gnupg/aptlytest.gpg")])
 
         if hasattr(self, "fixtureCmds"):
             for cmd in self.fixtureCmds:

system/run-aptly-cmd

@@ -1,17 +1,10 @@
 #!/bin/sh
 
-# cleanup
-rm -rf /app/tmp
-rm -rf /tmp/aptly*
-
-usermod -u `stat -c %u /app` aptly >/dev/null
-chown -R `stat -c %u /app` /var/lib/aptly
-
-# use same /home/runner dir as in github workflow
-chown -R `stat -c %u /app` /home/runner
-
 cmd=$@
-if [ -z "$cmd" ]; then
+test -z "$cmd" && cmd="bash"
-    cmd="bash"
+
-fi
+usermod -u `stat -c %u /work/src` aptly >/dev/null
-su aptly -c "cd /app; export HOME=/home/runner; export PATH=\$PATH:/app/build; $cmd"
+chown -R `stat -c %u /work/src` /var/lib/aptly
+
+cd /work/src
+exec sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go $cmd

system/run-golangci-lint

@@ -1,17 +1,6 @@
-#!/bin/sh
+#!/bin/sh -e
-
-# cleanup
-rm -rf /app/tmp
-rm -rf /tmp/aptly*
 
 usermod -u `stat -c %u /app` aptly >/dev/null
 chown -R `stat -c %u /app` /var/lib/aptly
 
-# use same /home/runner dir as in github workflow
+su aptly -c "set -e; cd /app; export GOPATH=/app/.go; go install github.com/golangci/golangci-lint/cmd/golangci-lint@$GOLANGCI_LINT_VERSION; PATH=\$PATH:/app/.go/bin golangci-lint run"
-chown -R `stat -c %u /app` /home/runner
-
-cmd=$@
-if [ -z "$cmd" ]; then
-    cmd="bash"
-fi
-su aptly -c "cd /app; GOPATH=/app/.go go install github.com/golangci/golangci-lint/cmd/golangci-lint@$GOLANGCI_LINT_VERSION; PATH=\$PATH:/app/.go/bin golangci-lint run"

system/run-system-tests

@@ -7,6 +7,4 @@ rm -rf /tmp/aptly*
 usermod -u `stat -c %u /app` aptly >/dev/null
 chown -R `stat -c %u /app` /var/lib/aptly
 
-# use same /home/runner dir as in github workflow
+su - aptly -c "cd /app; export GOPATH=/app/.go; go mod tidy; make docker-test TEST=$@"
-chown -R `stat -c %u /app` /home/runner
-su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make docker-test TEST=$@"

system/run-unit-tests

@@ -5,10 +5,7 @@ rm -rf /app/tmp
 rm -rf /tmp/aptly*
 
 mkdir -p /srv
-
 usermod -u `stat -c %u /app` aptly >/dev/null
 chown -R `stat -c %u /app` /var/lib/aptly /srv
 
-# use same /home/runner dir as in github workflow
+su - aptly -c "cd /app; export export GOPATH=/app/.go; go mod tidy; make test"
-chown -R `stat -c %u /app` /home/runner
-su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make test"

system/t04_mirror/CreateMirror18Test_gold

@@ -1,5 +1,4 @@
 Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
-gpgv: can't allocate lock for '/home/runner/.gnupg/ppa.gpg'
 gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC
 gpgv:                using RSA key 5BFCD481D86D5824470E469F9000B1C3A01F726C
 gpgv: Good signature from "Launchpad PPA for Anton Gladky"

system/t04_mirror/CreateMirror30Test_gold

@@ -1,4 +1,4 @@
-opengpg: failure opening keyring '/home/runner/.gnupg/aptlytest.gpg': open /home/runner/.gnupg/aptlytest.gpg: no such file or directory
+opengpg: failure opening keyring '${HOME}/.gnupg/aptlytest.gpg': open ${HOME}/.gnupg/aptlytest.gpg: no such file or directory
 
 Looks like your keyring with trusted keys is empty. You might consider importing some keys.
 Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease

system/t04_mirror/create.py

@@ -223,7 +223,8 @@ class CreateMirror18Test(BaseTest):
     }
 
     fixtureCmds = [
-        "gpg --no-default-keyring --keyring=ppa.gpg --keyserver=hkp://keyserver.ubuntu.com:80 --recv-keys 5BFCD481D86D5824470E469F9000B1C3A01F726C 02219381E9161C78A46CB2BFA5279A973B1F56C0"
+        "gpg --no-default-keyring --keyring=ppa.gpg --keyserver=hkp://keyserver.ubuntu.com:80 --recv-keys 5BFCD481D86D5824470E469F9000B1C3A01F726C 02219381E9161C78A46CB2BFA5279A973B1F56C0",
+        f"chmod 400 {os.path.join(os.environ['HOME'], '.gnupg/ppa.gpg')}"
     ]
     runCmd = "aptly mirror create -keyring=ppa.gpg mirror18 ppa:gladky-anton/gnuplot"
 

system/t06_publish/PublishSnapshot41Test_gold

@@ -5,7 +5,7 @@ Signing file 'Release' with gpg, please enter your passphrase when prompted:
 Clearsigning file 'Release' with gpg, please enter your passphrase when prompted:
 
 Snapshot snap41 has been successfully published.
-Please setup your webserver to serve directory '/home/runner/.aptly/public' with autoindexing.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
 Now you can add following line to apt sources:
   deb http://your-server/ buster/updates main
   deb-src http://your-server/ buster/updates main