ci: remove pip and virtualenv

- separate unit tests, benchmark, system tests, flake8

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ env:
 
 jobs:
   test:
-    name: "System Tests (Ubuntu 22.04)"
+    name: "Test (Ubuntu 22.04)"
     runs-on: ubuntu-22.04
     continue-on-error: false
     timeout-minutes: 30
@@ -33,7 +33,7 @@ jobs:
       - name: "Install packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
 
       - name: "Checkout repository"
         uses: actions/checkout@v4
@@ -41,6 +41,10 @@ jobs:
           # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
+      - name: "Run flake8"
+        run: |
+          make flake8
+
       - name: "Read go version from go.mod"
         run: |
           gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
@@ -53,23 +57,28 @@ jobs:
         with:
           go-version: ${{ steps.goversion.outputs.GOVER }}
 
-      - name: "Setup Python"
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.11'
-
-      - name: "Install Python packages"
-        run: |
-          pip install six packaging appdirs virtualenv
-          pip install -U pip setuptools
-          pip install -r system/requirements.txt
-
       - name: "Install Azurite"
         id: azuright
         uses: potatoqualitee/azuright@v1.1
         with:
           directory: ${{ runner.temp }}
 
+      - name: "Run Unit Tests"
+        env:
+          RUN_LONG_TESTS: 'yes'
+          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
+          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
+          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
+          COVERAGE_DIR=${{ runner.temp }} make test
+
+      - name: "Run Benchmark"
+        run: |
+          COVERAGE_DIR=${{ runner.temp }} make bench
+
       - name: "Run System Tests"
         env:
           RUN_LONG_TESTS: 'yes'
@@ -80,7 +89,7 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           sudo mkdir -p /srv ; sudo chown runner /srv
-          COVERAGE_DIR=${{ runner.temp }} make all
+          COVERAGE_DIR=${{ runner.temp }} make system-test
 
       - name: "Merge code coverage"
         run: |

Makefile

@@ -17,8 +17,6 @@ RELEASE=no
 help:  ## Print this help
 	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-all: prepare test bench check system-test  # used for system tests in ci
-
 prepare:  ## Install go module dependencies
 	# set version
 	@make version > VERSION
@@ -31,24 +29,12 @@ prepare:  ## Install go module dependencies
 	go mod tidy -v
 	go generate
 
-check: system/env
-ifeq ($(RUN_LONG_TESTS), yes)
-	system/env/bin/flake8
-endif
-
 install:
 	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
 	go generate
 	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-system/env: system/requirements.txt
+system-test: install ## Run system tests in github CI
-ifeq ($(RUN_LONG_TESTS), yes)
-	rm -rf system/env
-	$(PYTHON) -m venv system/env
-	system/env/bin/pip install -r system/requirements.txt
-endif
-
-system-test: install system/env  ## Run system tests in github CI
 ifeq ($(RUN_LONG_TESTS), yes)
 	go generate
 	# install etcd
@@ -61,7 +47,7 @@ ifeq ($(RUN_LONG_TESTS), yes)
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db.xz; xz -d etcd.db.xz
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
+	PATH=$(BINPATH)/:$(PATH) && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
 endif
 
 docker-test: ## Run system tests
@@ -203,7 +189,7 @@ docker-deb:  ## Build debian packages in docker container
 docker-unit-tests:  ## Run unit tests in docker container
 	@docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-unit-tests
 
-docker-system-tests:  ## Run system tests in docker container (add TEST=t04_mirror to run only specific tests)
+docker-system-tests:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
 	@docker run -it --rm -v ${PWD}:/app aptly-dev /app/system/run-system-tests $(TEST)
 
 docker-dev-server:  ## Run development server (auto recompiling) on http://localhost:3142
@@ -223,4 +209,4 @@ clean:  ## remove local build and module cache
 	rm -rf build/ docs/ obj-*-linux-gnu*
 	rm -f unit.out aptly.test
 
-.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server
+.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server flake8

system/lib.py

@@ -103,7 +103,7 @@ class DotFinder(object):
     def find_dot(self, executables):
         for executable in executables:
             try:
-                subprocess.check_output([executable, "-V"], text=True)
+                subprocess.check_output([executable, "-V"], text=True, stderr=subprocess.DEVNULL)
                 return executable
             except Exception:
                 pass