Capture coverage of integration tests

To capture the coverage also for the integration tests,
a test only executing the cmd.Run function is used.

The test always exits with code 0 and prints the
real exit code to stdout. Otherwise no coverage
report is generated.

Those changes enable a more accurate coverage report
for future contributions.

.github/workflows/ci.yml

@@ -51,10 +51,15 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Setup Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
 
+      - name: Get aptly version
+        run: |
+           make version
+           go generate
+
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
@@ -82,9 +87,9 @@ jobs:
         with:
           directory: ${{ runner.temp }}
 
-      - name: Get aptly version
+      - name: Create temp dir
         run: |
-           make version
+          echo "COVERAGE_DIR=$(mktemp -d)" >> $GITHUB_ENV
 
       - name: Make
         env:
@@ -95,8 +100,14 @@ jobs:
         run: |
           make
 
+      - name: Merge code coverage
+        if: matrix.run_long_tests == 'yes'
+        run: |
+          go install github.com/wadey/gocovmerge@latest
+          ~/go/bin/gocovmerge unit.out $COVERAGE_DIR/*.out > coverage.txt
+
       - name: Upload code coverage
-        if: matrix.run_long_tests
+        if: matrix.run_long_tests == 'yes'
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -37,6 +37,9 @@ man/aptly.1.ronn
 system/env/
 
 # created by make build for release artifacts
+VERSION
+aptly.test
+
 build/
 
 pgp/keyrings/aptly2*.gpg

Makefile

@@ -1,20 +1,17 @@
 GOVERSION=$(shell go version | awk '{print $$3;}')
-ifdef TRAVIS_TAG
-	TAG=$(TRAVIS_TAG)
-else
 TAG="$(shell git describe --tags --always)"
-endif
 VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
 PACKAGES=context database deb files gpg http query swift s3 utils
 PYTHON?=python3
 TESTS?=
 BINPATH?=$(GOPATH)/bin
 RUN_LONG_TESTS?=yes
+COVERAGE_DIR?=$(shell mktemp -d)
 
 all: modules test bench check system-test
 
 prepare:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.45.0
 
 modules:
 	go mod download
@@ -31,7 +28,8 @@ ifeq ($(RUN_LONG_TESTS), yes)
 endif
 
 install:
-	go install -v -ldflags "-X main.Version=$(VERSION)"
+	go generate
+	go install -v
 
 system/env: system/requirements.txt
 ifeq ($(RUN_LONG_TESTS), yes)
@@ -42,13 +40,15 @@ endif
 
 system-test: install system/env
 ifeq ($(RUN_LONG_TESTS), yes)
+	go generate
+	go test -v -coverpkg="./..." -c -tags testruncli
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR)
 endif
 
 test:
-	go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
+	go test -v ./... -gocheck.v=true -coverprofile=unit.out
 
 bench:
 	go test -v ./deb -run=nothing -bench=. -benchmem

api/api_test.go

@@ -2,8 +2,6 @@ package api
 
 import (
 	"encoding/json"
-	ctx "github.com/aptly-dev/aptly/context"
-	"github.com/gin-gonic/gin"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -11,6 +9,9 @@ import (
 	"os"
 	"testing"
 
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+
 	"github.com/smira/flag"
 
 	. "gopkg.in/check.v1"

go.mod

@@ -1,6 +1,6 @@
 module github.com/aptly-dev/aptly
 
-go 1.15
+go 1.16
 
 require (
 	github.com/AlekSi/pointer v1.0.0

main.go

@@ -7,9 +7,12 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/cmd"
+
+	_ "embed"
 )
 
-// Version variable, filled in at link time
+//go:generate sh -c "make -s version | tr -d '\n' > VERSION"
+//go:embed VERSION
 var Version string
 
 func main() {

main_test.go

@@ -0,0 +1,63 @@
+//go:build testruncli
+// +build testruncli
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/cmd"
+)
+
+func filterOutTestArgs(args []string) (out []string) {
+	for _, arg := range args {
+		if !strings.Contains(arg, "-test.coverprofile") {
+			out = append(out, arg)
+		}
+	}
+	return
+}
+
+// redefine all the flags otherwise the go testing tool
+// is not able to parse them ...
+var _ = flag.Int("db-open-attempts", 10, "number of attempts to open DB if it's locked by other instance")
+var _ = flag.Bool("dep-follow-suggests", false, "when processing dependencies, follow Suggests")
+var _ = flag.Bool("dep-follow-source", false, "when processing dependencies, follow from binary to Source packages")
+var _ = flag.Bool("dep-follow-recommends", false, "when processing dependencies, follow Recommends")
+var _ = flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
+var _ = flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
+var _ = flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
+var _ = flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
+var _ = flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
+
+var _ = flag.String("cpuprofile", "", "write cpu profile to file")
+var _ = flag.String("memprofile", "", "write memory profile to this file")
+var _ = flag.String("memstats", "", "write memory stats periodically to this file")
+var _ = flag.Duration("meminterval", 100*time.Millisecond, "memory stats dump interval")
+
+var _ = flag.Bool("raw", false, "raw")
+var _ = flag.String("sort", "false", "sort")
+var _ = flag.Bool("json", false, "json")
+
+func TestRunMain(t *testing.T) {
+	if Version == "" {
+		Version = "unknown"
+	}
+
+	aptly.Version = Version
+
+	rand.Seed(time.Now().UnixNano())
+
+	args := filterOutTestArgs(os.Args[1:])
+	root := cmd.RootCommand()
+	root.UsageLine = "aptly"
+
+	fmt.Printf("EXIT: %d\n", cmd.Run(root, args, true))
+}

system/lib.py

@@ -19,6 +19,8 @@ import urllib.request
 import pprint
 import socketserver
 import http.server
+from uuid import uuid4
+from pathlib import Path
 import zlib
 
 
@@ -264,6 +266,10 @@ class BaseTest(object):
             command = string.Template(command).substitute(params)
             command = shlex.split(command)
 
+        if command[0] == "aptly":
+            aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test"
+            command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]]
+
         environ = os.environ.copy()
         environ["LC_ALL"] = "C"
         environ.update(self.environmentOverride)
@@ -272,14 +278,34 @@ class BaseTest(object):
     def run_cmd(self, command, expected_code=0):
         try:
             proc = self._start_process(command, stdout=subprocess.PIPE)
-            output, _ = proc.communicate()
+            raw_output, _ = proc.communicate()
+
+            returncodes = [proc.returncode]
+            is_aptly_command = False
+            if isinstance(command, str):
+                is_aptly_command = command.startswith("aptly")
+
+            if isinstance(command, list):
+                is_aptly_command = command[0] == "aptly"
+
+            if is_aptly_command:
+                # remove the last two rows as go tests always print PASS/FAIL and coverage in those
+                # two lines. This would otherwise fail the tests as they would not match gold
+                output, _, returncode = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output.decode("utf-8"))[0]
+
+                output = output.encode()
+                returncodes.append(int(returncode))
+
+            else:
+                output = raw_output
+
             if expected_code is not None:
-                if proc.returncode != expected_code:
+                if expected_code not in returncodes:
                     raise Exception("exit code %d != %d (output: %s)" % (
-                        proc.returncode, expected_code, output))
+                        proc.returncode, expected_code, raw_output))
             return output
         except Exception as e:
-            raise Exception("Running command %s failed: %s" %
+            raise Exception("Running command '%s' failed: %s" %
                             (command, str(e)))
 
     def gold_processor(self, gold):

system/run.py

@@ -7,6 +7,7 @@ import inspect
 import fnmatch
 import re
 import sys
+from tempfile import mkdtemp
 import traceback
 import random
 import subprocess
@@ -42,7 +43,7 @@ def walk_modules(package):
         yield importlib.import_module(package + "." + name)
 
 
-def run(include_long_tests=False, capture_results=False, tests=None, filters=None):
+def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None):
     """
     Run system test.
     """
@@ -51,6 +52,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     fails = []
     numTests = numFailed = numSkipped = 0
     lastBase = None
+    if not coverage_dir:
+        coverage_dir = mkdtemp(suffix="aptly-coverage")
 
     for test in tests:
         for testModule in walk_modules(test):
@@ -95,6 +98,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
 
                 try:
                     t.captureResults = capture_results
+                    t.coverage_dir = coverage_dir
                     t.test()
                 except Exception:
                     numFailed += 1
@@ -110,6 +114,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     if lastBase is not None:
         lastBase.shutdown_class()
 
+    print("COVERAGE_RESULTS: %s" % coverage_dir)
+
     print("TESTS: %d SUCCESS: %d FAIL: %d SKIP: %d" % (
         numTests, numTests - numFailed, numFailed, numSkipped))
 
@@ -149,6 +155,7 @@ if __name__ == "__main__":
     random.seed()
     include_long_tests = False
     capture_results = False
+    coverage_dir = None
     tests = None
     args = sys.argv[1:]
 
@@ -157,6 +164,9 @@ if __name__ == "__main__":
             include_long_tests = True
         elif args[0] == "--capture":
             capture_results = True
+        elif args[0] == "--coverage-dir":
+            coverage_dir = args[1]
+            args = args[1:]
 
         args = args[1:]
 
@@ -169,4 +179,4 @@ if __name__ == "__main__":
         else:
             filters.append(arg)
 
-    run(include_long_tests, capture_results, tests, filters)
+    run(include_long_tests, capture_results, tests, filters, coverage_dir)