diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a58ddb7..f5b85088 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,15 @@ jobs: uses: actions/checkout@v2 - name: Setup Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v3 with: go-version: ${{ matrix.go }} + - name: Get aptly version + run: | + make version + go generate + - name: golangci-lint uses: golangci/golangci-lint-action@v2 with: @@ -82,9 +87,9 @@ jobs: with: directory: ${{ runner.temp }} - - name: Get aptly version + - name: Create temp dir run: | - make version + echo "COVERAGE_DIR=$(mktemp -d)" >> $GITHUB_ENV - name: Make env: @@ -95,8 +100,14 @@ jobs: run: | make + - name: Merge code coverage + if: matrix.run_long_tests == 'yes' + run: | + go install github.com/wadey/gocovmerge@latest + ~/go/bin/gocovmerge unit.out $COVERAGE_DIR/*.out > coverage.txt + - name: Upload code coverage - if: matrix.run_long_tests + if: matrix.run_long_tests == 'yes' uses: codecov/codecov-action@v2 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 347cbc1d..76902418 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,9 @@ man/aptly.1.ronn system/env/ # created by make build for release artifacts +VERSION +aptly.test + build/ pgp/keyrings/aptly2*.gpg diff --git a/Makefile b/Makefile index 1acb5035..c26edf77 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,17 @@ GOVERSION=$(shell go version | awk '{print $$3;}') -ifdef TRAVIS_TAG - TAG=$(TRAVIS_TAG) -else - TAG="$(shell git describe --tags --always)" -endif +TAG="$(shell git describe --tags --always)" VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g') PACKAGES=context database deb files gpg http query swift s3 utils PYTHON?=python3 TESTS?= BINPATH?=$(GOPATH)/bin RUN_LONG_TESTS?=yes +COVERAGE_DIR?=$(shell mktemp -d) all: modules test bench check system-test prepare: - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.45.0 modules: go mod download @@ -31,7 +28,8 @@ ifeq ($(RUN_LONG_TESTS), yes) endif install: - go install -v -ldflags "-X main.Version=$(VERSION)" + go generate + go install -v system/env: system/requirements.txt ifeq ($(RUN_LONG_TESTS), yes) @@ -42,13 +40,15 @@ endif system-test: install system/env ifeq ($(RUN_LONG_TESTS), yes) + go generate + go test -v -coverpkg="./..." -c -tags testruncli if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi - PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) + PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) endif test: - go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic + go test -v ./... -gocheck.v=true -coverprofile=unit.out bench: go test -v ./deb -run=nothing -bench=. -benchmem diff --git a/api/api_test.go b/api/api_test.go index 2e55638e..fd8105f0 100644 --- a/api/api_test.go +++ b/api/api_test.go @@ -2,8 +2,6 @@ package api import ( "encoding/json" - ctx "github.com/aptly-dev/aptly/context" - "github.com/gin-gonic/gin" "io" "io/ioutil" "net/http" @@ -11,6 +9,9 @@ import ( "os" "testing" + ctx "github.com/aptly-dev/aptly/context" + "github.com/gin-gonic/gin" + "github.com/smira/flag" . "gopkg.in/check.v1" diff --git a/go.mod b/go.mod index 4c2e85ad..00481e74 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/aptly-dev/aptly -go 1.15 +go 1.16 require ( github.com/AlekSi/pointer v1.0.0 diff --git a/main.go b/main.go index 80093443..2503571a 100644 --- a/main.go +++ b/main.go @@ -7,9 +7,12 @@ import ( "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/cmd" + + _ "embed" ) -// Version variable, filled in at link time +//go:generate sh -c "make -s version | tr -d '\n' > VERSION" +//go:embed VERSION var Version string func main() { diff --git a/main_test.go b/main_test.go new file mode 100644 index 00000000..35208ed3 --- /dev/null +++ b/main_test.go @@ -0,0 +1,63 @@ +//go:build testruncli +// +build testruncli + +package main + +import ( + "flag" + "fmt" + "math/rand" + "os" + "strings" + "testing" + "time" + + "github.com/aptly-dev/aptly/aptly" + "github.com/aptly-dev/aptly/cmd" +) + +func filterOutTestArgs(args []string) (out []string) { + for _, arg := range args { + if !strings.Contains(arg, "-test.coverprofile") { + out = append(out, arg) + } + } + return +} + +// redefine all the flags otherwise the go testing tool +// is not able to parse them ... +var _ = flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance") +var _ = flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests") +var _ = flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages") +var _ = flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends") +var _ = flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'") +var _ = flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs") +var _ = flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available") +var _ = flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)") +var _ = flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)") + +var _ = flag.String("cpuprofile", "", "write cpu profile to file") +var _ = flag.String("memprofile", "", "write memory profile to this file") +var _ = flag.String("memstats", "", "write memory stats periodically to this file") +var _ = flag.Duration("meminterval", 100*time.Millisecond, "memory stats dump interval") + +var _ = flag.Bool("raw", false, "raw") +var _ = flag.String("sort", "false", "sort") +var _ = flag.Bool("json", false, "json") + +func TestRunMain(t *testing.T) { + if Version == "" { + Version = "unknown" + } + + aptly.Version = Version + + rand.Seed(time.Now().UnixNano()) + + args := filterOutTestArgs(os.Args[1:]) + root := cmd.RootCommand() + root.UsageLine = "aptly" + + fmt.Printf("EXIT: %d\n", cmd.Run(root, args, true)) +} diff --git a/system/lib.py b/system/lib.py index 013cfc3a..992dfd11 100644 --- a/system/lib.py +++ b/system/lib.py @@ -19,6 +19,8 @@ import urllib.request import pprint import socketserver import http.server +from uuid import uuid4 +from pathlib import Path import zlib @@ -264,6 +266,10 @@ class BaseTest(object): command = string.Template(command).substitute(params) command = shlex.split(command) + if command[0] == "aptly": + aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test" + command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]] + environ = os.environ.copy() environ["LC_ALL"] = "C" environ.update(self.environmentOverride) @@ -272,14 +278,34 @@ class BaseTest(object): def run_cmd(self, command, expected_code=0): try: proc = self._start_process(command, stdout=subprocess.PIPE) - output, _ = proc.communicate() + raw_output, _ = proc.communicate() + + returncodes = [proc.returncode] + is_aptly_command = False + if isinstance(command, str): + is_aptly_command = command.startswith("aptly") + + if isinstance(command, list): + is_aptly_command = command[0] == "aptly" + + if is_aptly_command: + # remove the last two rows as go tests always print PASS/FAIL and coverage in those + # two lines. This would otherwise fail the tests as they would not match gold + output, _, returncode = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output.decode("utf-8"))[0] + + output = output.encode() + returncodes.append(int(returncode)) + + else: + output = raw_output + if expected_code is not None: - if proc.returncode != expected_code: + if expected_code not in returncodes: raise Exception("exit code %d != %d (output: %s)" % ( - proc.returncode, expected_code, output)) + proc.returncode, expected_code, raw_output)) return output except Exception as e: - raise Exception("Running command %s failed: %s" % + raise Exception("Running command '%s' failed: %s" % (command, str(e))) def gold_processor(self, gold): diff --git a/system/run.py b/system/run.py index e27377d0..1ba5c1ea 100755 --- a/system/run.py +++ b/system/run.py @@ -7,6 +7,7 @@ import inspect import fnmatch import re import sys +from tempfile import mkdtemp import traceback import random import subprocess @@ -42,7 +43,7 @@ def walk_modules(package): yield importlib.import_module(package + "." + name) -def run(include_long_tests=False, capture_results=False, tests=None, filters=None): +def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None): """ Run system test. """ @@ -51,6 +52,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non fails = [] numTests = numFailed = numSkipped = 0 lastBase = None + if not coverage_dir: + coverage_dir = mkdtemp(suffix="aptly-coverage") for test in tests: for testModule in walk_modules(test): @@ -95,6 +98,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non try: t.captureResults = capture_results + t.coverage_dir = coverage_dir t.test() except Exception: numFailed += 1 @@ -110,6 +114,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if lastBase is not None: lastBase.shutdown_class() + print("COVERAGE_RESULTS: %s" % coverage_dir) + print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % ( numTests, numTests - numFailed, numFailed, numSkipped)) @@ -149,6 +155,7 @@ if __name__ == "__main__": random.seed() include_long_tests = False capture_results = False + coverage_dir = None tests = None args = sys.argv[1:] @@ -157,6 +164,9 @@ if __name__ == "__main__": include_long_tests = True elif args[0] == "--capture": capture_results = True + elif args[0] == "--coverage-dir": + coverage_dir = args[1] + args = args[1:] args = args[1:] @@ -169,4 +179,4 @@ if __name__ == "__main__": else: filters.append(arg) - run(include_long_tests, capture_results, tests, filters) + run(include_long_tests, capture_results, tests, filters, coverage_dir)