From c248dc18036c6571d61e3e48563cc4bab4554537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Roth?= Date: Sun, 21 Apr 2024 16:42:51 +0200 Subject: [PATCH] github CI: use dpkg-buildpackage for building debian packages - use go 1.19 - Makefile: improve unit test output - cleanup: remove travis --- .github/workflows/ci.yml | 146 +++++++++++++++++----------- .github/workflows/golangci-lint.yml | 2 +- .goxc.json | 29 +----- .travis.yml | 106 -------------------- Makefile | 28 +++--- 5 files changed, 110 insertions(+), 201 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f37d730..9df517af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,3 @@ -# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml - name: CI on: @@ -13,14 +11,14 @@ on: defaults: run: # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell - shell: bash --noprofile --norc -eo pipefail -x {0} + shell: bash --noprofile --norc -eo pipefail {0} env: DEBIAN_FRONTEND: noninteractive jobs: build: - name: test + name: "System Tests (Ubuntu 22.04)" runs-on: ubuntu-22.04 continue-on-error: false timeout-minutes: 30 @@ -30,39 +28,41 @@ jobs: BOTO_CONFIG: /dev/null GO111MODULE: "on" GOPROXY: "https://proxy.golang.org" - GOVER: '1.21' + GOVER: '1.19' steps: - - name: Checkout repository - uses: actions/checkout@v2 + - name: "Install packages" + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts - - name: Setup Go + - name: "Checkout repository" + uses: actions/checkout@v3 + with: + # fetch the whole repo for `git describe` to work + fetch-depth: 0 + + - name: "Setup Go" uses: actions/setup-go@v3 with: go-version: ${{ env.GOVER }} - - name: Get aptly version + - name: "Get aptly version" run: | make version - go generate - - name: Setup Python + - name: "Setup Python" uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Install O/S packages - run: | - sudo apt-get update - sudo apt-get install -y graphviz gnupg2 gpgv2 git gcc make - - - name: Install Python packages + - name: "Install Python packages" run: | pip install six packaging appdirs virtualenv pip install -U pip setuptools pip install -r system/requirements.txt - - name: Install Azurite + - name: "Install Azurite" id: azuright uses: potatoqualitee/azuright@v1.1 with: @@ -94,70 +94,72 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.txt - release: + ci-debian: + name: "CI Build (Deb)" needs: build runs-on: ubuntu-latest strategy: fail-fast: false matrix: - name: [Ubuntu-20, Ubuntu-22, Debian-10, Debian-11, Debian-12] + name: ["Debian 12", "Debian 11", "Debian 10", "Ubuntu 22.04", "Ubuntu 20.04"] include: - - name: Ubuntu-20 - image: "ubuntu:20.04" - suite: focal - GOVER: '1.21' - install: "make ca-certificates git curl" - - name: Ubuntu-22 - image: "ubuntu:22.04" - suite: jammy - install: "make ca-certificates git curl golang" - - name: Debian-10 - image: "debian:buster" - suite: buster - GOVER: '1.21' - install: "make ca-certificates git curl" - - name: Debian-11 - image: "debian:bullseye" - suite: bullseye - GOVER: '1.21' - install: "make ca-certificates git curl" - - name: Debian-12 - image: "debian:bookworm" + - name: "Debian 12" suite: bookworm - install: "make ca-certificates git curl golang" + image: debian:bookworm + - name: "Debian 10" + suite: buster + image: debian:buster + - name: "Debian 11" + suite: bullseye + image: debian:bullseye + - name: "Ubuntu 22.04" + suite: jammy + image: ubuntu:22.04 + - name: "Ubuntu 20.04" + suite: focal + image: ubuntu:20.04 container: image: ${{ matrix.image }} env: APT_LISTCHANGES_FRONTEND: none DEBIAN_FRONTEND: noninteractive steps: - - name: Install O/S packages + - name: "Install packages" run: | apt-get update - apt-get install -y --no-install-recommends ${{ matrix.install }} + apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf git config --global --add safe.directory "$GITHUB_WORKSPACE" - echo GOVER: ${{ env.GOVER }} - - name: Checkout repository + - name: "Checkout repository" uses: actions/checkout@v3 with: - # fetch the whole repo for `git describe` to - # work and get the nightly verion + # fetch the whole repo for `git describe` to work fetch-depth: 0 - - name: Setup Go - if: ${{ matrix.GOVER != '' }} + - name: "Read go version from go.mod" + run: | + echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT + id: goversion + + - name: "Setup Go" uses: actions/setup-go@v3 with: - go-version: ${{ matrix.GOVER }} + go-version: ${{ steps.goversion.outputs.GOVER }} - - name: Make Release + - name: "Build Debian packages" env: GOBIN: /usr/local/bin run: | - make release + GOPATH=$PWD/.go go generate -v + DEBEMAIL="CI " dch -v `make version` "CI build" + dpkg-buildpackage -us -uc -b -d --host-arch=amd64 + dpkg-buildpackage -us -uc -b -d --host-arch=i386 + dpkg-buildpackage -us -uc -b -d --host-arch=arm64 + dpkg-buildpackage -us -uc -b -d --host-arch=armhf + mkdir -p build && mv ../*.deb build/ + cd build && ls -l *.deb - - name: Publish nightly release to aptly + - name: "Publish CI release to aptly" if: github.ref == 'refs/heads/master' env: APTLY_USER: ${{ secrets.APTLY_USER }} @@ -165,10 +167,44 @@ jobs: run: | ./upload-artifacts.sh nightly ${{ matrix.suite }} - - name: Publish release to aptly + - name: "Publish release to aptly" if: startsWith(github.event.ref, 'refs/tags') env: APTLY_USER: ${{ secrets.APTLY_USER }} APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }} run: | ./upload-artifacts.sh release ${{ matrix.suite }} + + ci-other: + name: "CI Build (FreeBSD, MacOS, Linux)" + needs: build + runs-on: ubuntu-latest + steps: + - name: "Install packages" + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends make ca-certificates git curl devscripts + git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: "Checkout repository" + uses: actions/checkout@v3 + with: + # fetch the whole repo for `git describe` to work + fetch-depth: 0 + + - name: "Read go version from go.mod" + run: | + echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT + id: goversion + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: ${{ steps.goversion.outputs.GOVER }} + + - name: "Build goxc packages" + env: + GOBIN: /usr/local/bin + run: | + go version + make release diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 609a8214..837e8cb9 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.21' + go-version: '1.19' cache: false - name: Create VERSION file run: | diff --git a/.goxc.json b/.goxc.json index 60f1940c..0adf4893 100644 --- a/.goxc.json +++ b/.goxc.json @@ -6,35 +6,8 @@ "go-test", "go-vet" ], - "TasksAppend": [ - "bintray" - ], - "TaskSettings": { - "debs": { - "metadata": { - "maintainer": "Andrey Smirnov", - "maintainer-email": "me@smira.ru", - "description": "Debian repository management tool" - }, - "metadata-deb": { - "License": "MIT", - "Homepage": "https://www.aptly.info/", - "Depends": "bzip2, xz-utils, gnupg, gpgv", - "Suggests": "graphviz" - }, - "other-mapped-files": { - "/": "root/" - } - }, - "bintray": { - "repository": "aptly", - "subject": "smira", - "package": "aptly", - "downloadspage": "bintray.md" - } - }, "ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1", - "BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64", + "BuildConstraints": "linux,386 linux,amd64 linux,arm linux,arm64 darwin,amd64 freebsd,386 freebsd,amd64", "MainDirsExclude": "_man,vendor", "BuildSettings": { "LdFlagsXVars": { diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 574a8f4e..00000000 --- a/.travis.yml +++ /dev/null @@ -1,106 +0,0 @@ -dist: xenial -sudo: required - -language: go - -go_import_path: github.com/aptly-dev/aptly - -addons: - apt: - packages: - - python-virtualenv - - graphviz - - gnupg2 - - gpgv2 - -env: - global: - - secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8=" - - secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE=" - - NO_FTP_ACCESS: "yes" - - BOTO_CONFIG: /dev/null - - GO111MODULE: "on" - - GOPROXY: https://proxy.golang.org - -matrix: - allow_failures: - - go: master - env: RUN_LONG_TESTS=no - fast_finish: true - include: - - go: 1.11.x - env: RUN_LONG_TESTS=no - - go: 1.12.x - env: RUN_LONG_TESTS=no - - go: 1.13.x - env: RUN_LONG_TESTS=no - - go: 1.14.x - env: RUN_LONG_TESTS=yes - - go: 1.15.x - env: - - RUN_LONG_TESTS=yes - - DEPLOY_BINARIES=yes - - APTLY_USER=aptly - - secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o=" - - go: master - env: RUN_LONG_TESTS=no - -before_install: - - virtualenv system/env - - . system/env/bin/activate - - pip install six packaging appdirs - - pip install -U pip setuptools - - pip install -r system/requirements.txt - - make version - -install: - - make prepare - -after_success: - - bash <(curl -s https://codecov.io/bash) - -notifications: - webhooks: - urls: - - "https://webhooks.gitter.im/e/c691da114a41eed6ec45" - on_success: change # options: [always|never|change] default: always - on_failure: always # options: [always|never|change] default: always - on_start: false # default: false - -before_deploy: - - make release - -deploy: - - provider: releases - api_key: - secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg= - file_glob: true - file: build/* - skip_cleanup: true - on: - tags: true - condition: "$DEPLOY_BINARIES = yes" - - provider: s3 - access_key_id: - secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM=" - secret_access_key: - secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg=" - bucket: aptly-nightly - skip_cleanup: true - acl: public-read - local_dir: build - on: - branch: master - condition: "$DEPLOY_BINARIES = yes" - - provider: script - script: bash upload-artifacts.sh nightly - skip_cleanup: true - on: - branch: master - condition: "$DEPLOY_BINARIES = yes" - - provider: script - script: bash upload-artifacts.sh release - skip_cleanup: true - on: - tags: true - condition: "$DEPLOY_BINARIES = yes" diff --git a/Makefile b/Makefile index cbb49ed1..2b31b7a3 100644 --- a/Makefile +++ b/Makefile @@ -14,12 +14,13 @@ COVERAGE_DIR?=$(shell mktemp -d) help: ## Print this help @grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -all: modules test bench check system-test +all: prepare test bench check system-test -modules: ## Install go module dependencies +prepare: ## Install go module dependencies go mod download go mod verify go mod tidy -v + go generate dev: PATH=$(BINPATH)/:$(PATH) @@ -69,12 +70,16 @@ docker-test: ## Run system tests export APTLY_VERSION=$(VERSION); \ $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) -test: ## Run unit tests +test: prepare ## Run unit tests @test -d /srv/etcd || system/t13_etcd/install-etcd.sh - @system/t13_etcd/start-etcd.sh & - @echo Running go test - go test -v ./... -gocheck.v=true -coverprofile=unit.out - @kill `cat /tmp/etcd.pid` + @echo "\nStarting etcd ..." + @mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 & + @echo "\nRunning go test ..." + go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret + @echo "\nStopping etcd ..." + @pid=`cat /tmp/etcd.pid`; kill $$pid + @rm -f /tmp/etcd-data/etcd.log + @ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret bench: go test -v ./deb -run=nothing -bench=. -benchmem @@ -90,14 +95,15 @@ goxc: dev cp completion.d/aptly root/etc/bash_completion.d/ cp completion.d/_aptly root/usr/share/zsh/vendor-completions/ gzip root/usr/share/man/man1/aptly.1 - go generate - goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS) + GOPATH=$(PWD)/.go go generate + GOPATH=$(PWD)/.go goxc -pv=$(VERSION) -max-processors=4 $(GOXC_OPTS) -release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin +release: GOXC_OPTS=-tasks-=go-vet,go-test,rmbin release: goxc rm -rf build/ mkdir -p build/ mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/ + ls -l build/ man: ## Create man pages make -C man @@ -146,4 +152,4 @@ clean: ## remove local build and module cache test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ rm -rf build/ docs/ obj-x86_64-linux-gnu/ -.PHONY: help man modules version release goxc docker-build-aptly-dev docker-system-tests docker-unit-tests docker-lint docker-build build docker-aptly clean +.PHONY: help man prepare version release goxc docker-build-aptly-dev docker-system-tests docker-unit-tests docker-lint docker-build build docker-aptly clean