diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8b61e9c..fa9dd64c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,10 +30,10 @@ jobs: GOPROXY: "https://proxy.golang.org" steps: - - name: "Install Packages" + - name: "Install Test Packages" run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 + sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime - name: "Checkout Repository" uses: actions/checkout@v4 @@ -139,7 +139,7 @@ jobs: APT_LISTCHANGES_FRONTEND: none DEBIAN_FRONTEND: noninteractive steps: - - name: "Install packages" + - name: "Install Build Packages" run: | apt-get update apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \ diff --git a/Makefile b/Makefile index d998f34c..ffe2e8a3 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,9 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) +# Unit Tests and some sysmte tests rely on expired certificates, turn back the time +export TEST_FAKETIME := 2025-01-02 03:04:05 + # export CAPUTRE=1 for regenrating test gold files ifeq ($(CAPTURE),1) CAPTURE_ARG := --capture @@ -86,11 +89,11 @@ install: # go install -v @out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi -test: prepare swagger etcd-install ## Run unit tests +test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify which tests to run) @echo "\e[33m\e[1mStarting etcd ...\e[0m" @mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 & @echo "\e[33m\e[1mRunning go test ...\e[0m" - go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret + faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret @echo "\e[33m\e[1mStopping etcd ...\e[0m" @pid=`cat /tmp/etcd.pid`; kill $$pid @rm -f /tmp/aptly-etcd-data/etcd.log @@ -104,7 +107,7 @@ system-test: prepare swagger etcd-install ## Run system tests if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) # Run system tests - PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) + PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) bench: @echo "\e[33m\e[1mRunning benchmark ...\e[0m" @@ -176,13 +179,13 @@ docker-shell: ## Run aptly and other commands in docker container docker-deb: ## Build debian packages in docker container @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 -docker-unit-test: ## Run unit tests in docker container +docker-unit-test: ## Run unit tests in docker container (add TEST=regex to specify which tests to run) @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ azurite-start \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ - test \ + test TEST=$(TEST) \ azurite-stop docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) diff --git a/api/gpg.go b/api/gpg.go index 94891e93..47ba1d92 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -13,26 +13,34 @@ import ( ) type gpgAddKeyParams struct { - // Keyserver, when downloading GpgKeyIDs - Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"` - // GpgKeyIDs to download from Keyserver, comma separated list - GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500,8B48AD6246925553"` - // Armored gpg public ket, instead of downloading from keyserver - GpgKeyArmor string `json:"GpgKeyArmor" example:""` // Keyring for adding the keys (default: trustedkeys.gpg) Keyring string `json:"Keyring" example:"trustedkeys.gpg"` + + // Add ASCII armored gpg public key, do not download from keyserver + GpgKeyArmor string `json:"GpgKeyArmor" example:""` + + // Keyserver to download keys provided in `GpgKeyID` + Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"` + // Keys do download from `Keyserver`, separated by space + GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500 8B48AD6246925553"` } // @Summary Add GPG Keys // @Description **Adds GPG keys to aptly keyring** // @Description // @Description Add GPG public keys for veryfing remote repositories for mirroring. +// @Description +// @Description Keys can be added in two ways: +// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty) +// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty) +// @Description // @Tags Mirrors +// @Consume json +// @Param request body gpgAddKeyParams true "Parameters" // @Produce json // @Success 200 {object} string "OK" // @Failure 400 {object} Error "Bad Request" -// @Failure 404 {object} Error "Not Found" -// @Router /api/gpg [post] +// @Router /api/gpg/key [post] func apiGPGAddKey(c *gin.Context) { b := gpgAddKeyParams{} if c.Bind(&b) != nil { diff --git a/api/repos.go b/api/repos.go index dcd701d1..0ced5efd 100644 --- a/api/repos.go +++ b/api/repos.go @@ -107,9 +107,9 @@ type repoCreateParams struct { // @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""} // @Description ``` // @Tags Repos -// @Produce json // @Consume json // @Param request body repoCreateParams true "Parameters" +// @Produce json // @Success 201 {object} deb.LocalRepo // @Failure 404 {object} Error "Source snapshot not found" // @Failure 409 {object} Error "Local repo already exists" @@ -178,8 +178,10 @@ type reposEditParams struct { // @Summary Update Repository // @Description **Update local repository meta information** // @Tags Repos -// @Produce json +// @Param name path string true "Repository name" +// @Consume json // @Param request body reposEditParams true "Parameters" +// @Produce json // @Success 200 {object} deb.LocalRepo "msg" // @Failure 404 {object} Error "Not Found" // @Failure 500 {object} Error "Internal Server Error" @@ -231,8 +233,8 @@ func apiReposEdit(c *gin.Context) { // @Summary Get Repository Info // @Description Returns basic information about local repository. // @Tags Repos -// @Produce json // @Param name path string true "Repository name" +// @Produce json // @Success 200 {object} deb.LocalRepo // @Failure 404 {object} Error "Repository not found" // @Router /api/repos/{name} [get] @@ -254,9 +256,10 @@ func apiReposShow(c *gin.Context) { // @Description Cannot drop repos that are published. // @Description Needs force=1 to drop repos used as source by other repos. // @Tags Repos -// @Produce json +// @Param name path string true "Repository name" // @Param _async query bool false "Run in background and return task object" // @Param force query int false "force: 1 to enable" +// @Produce json // @Success 200 {object} task.ProcessReturnValue "Repo object" // @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Repo Conflict" @@ -306,12 +309,12 @@ func apiReposDrop(c *gin.Context) { // @Description ["Pi386 aptly 0.8 966561016b44ed80"] // @Description ``` // @Tags Repos -// @Produce json -// @Param name path string true "Snapshot to search" +// @Param name path string true "Repository name" // @Param q query string true "Package query (e.g Name%20(~%20matlab))" // @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query" // @Param format query string true "Set to 'details' to return extra info about each package" // @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name" +// @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Internal Server Error" @@ -406,9 +409,10 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li // @Description // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository. // @Tags Repos -// @Produce json +// @Param name path string true "Repository name" // @Param request body reposPackagesAddDeleteParams true "Parameters" // @Param _async query bool false "Run in background and return task object" +// @Produce json // @Success 200 {object} string "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -426,9 +430,11 @@ func apiReposPackagesAdd(c *gin.Context) { // @Description // @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages. // @Tags Repos -// @Produce json -// @Param request body reposPackagesAddDeleteParams true "Parameters" +// @Param name path string true "Repository name" // @Param _async query bool false "Run in background and return task object" +// @Consume json +// @Param request body reposPackagesAddDeleteParams true "Parameters" +// @Produce json // @Success 200 {object} string "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -608,8 +614,8 @@ type reposCopyPackageParams struct { // @Description Copies a package from a source to destination repository // @Tags Repos // @Produce json -// @Param name path string true "Source repo" -// @Param src path string true "Destination repo" +// @Param name path string true "Destination repo" +// @Param src path string true "Source repo" // @Param file path string true "File/packages to copy" // @Param _async query bool false "Run in background and return task object" // @Success 200 {object} task.ProcessReturnValue "msg" @@ -762,12 +768,15 @@ func apiReposCopyPackage(c *gin.Context) { // @Summary Include File from Directory // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos -// @Produce json +// @Param name path string true "Repository name" +// @Param dir path string true "Directory of packages" +// @Param file path string true "File/packages to include" // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" // @Param _async query bool false "Run in background and return task object" +// @Produce json // @Success 200 {object} string "msg" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir}/{file} [post] @@ -784,12 +793,14 @@ type reposIncludePackageFromDirResponse struct { // @Summary Include Directory // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Tags Repos -// @Produce json +// @Param name path string true "Repository name" +// @Param dir path string true "Directory of packages" // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" // @Param _async query bool false "Run in background and return task object" +// @Produce json // @Success 200 {object} reposIncludePackageFromDirResponse "Response" // @Failure 404 {object} Error "Not Found" // @Router /api/repos/{name}/include/{dir} [post] diff --git a/system/Dockerfile b/system/Dockerfile index e5e6661c..143569c0 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -6,7 +6,8 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b g++ python3-etcd3 python3-plyvel graphviz devscripts sudo dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu \ binutils-arm-linux-gnueabihf bash-completion zip ruby-dev lintian npm \ libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \ - gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu && \ + gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \ + faketime && \ apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/bash --home-dir /var/lib/aptly aptly diff --git a/system/lib.py b/system/lib.py index 3f95b102..f7761997 100644 --- a/system/lib.py +++ b/system/lib.py @@ -130,6 +130,7 @@ class BaseTest(object): sortOutput = False debugOutput = False EtcdServer = None + faketime = False aptlyDir = ".aptly" aptlyConfigFile = ".aptly.conf" @@ -311,6 +312,9 @@ class BaseTest(object): aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test" command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]] + if self.faketime: + command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command + environ = os.environ.copy() environ["LC_ALL"] = "C" environ.update(self.environmentOverride) diff --git a/system/run.py b/system/run.py index 599afe5f..4e73fb2d 100755 --- a/system/run.py +++ b/system/run.py @@ -50,6 +50,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non if not coverage_dir: coverage_dir = mkdtemp(suffix="aptly-coverage") + failed = False for test in tests: orig_stdout = sys.stdout orig_stderr = sys.stderr @@ -157,8 +158,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non t.shutdown() + if failed: + break + if failed: + break + sys.stdout = orig_stdout sys.stderr = orig_stderr + if failed: + break if lastBase is not None: lastBase.shutdown_class() diff --git a/system/s3_lib.py b/system/s3_lib.py index 4a52b9ad..6e622b2d 100644 --- a/system/s3_lib.py +++ b/system/s3_lib.py @@ -24,7 +24,9 @@ class S3Test(BaseTest): s3Overrides = {} def fixture_available(self): - return super(S3Test, self).fixture_available() and s3_conn is not None + return super(S3Test, self).fixture_available() and \ + 'AWS_SECRET_ACCESS_KEY' in os.environ and 'AWS_ACCESS_KEY_ID' in os.environ and \ + os.environ['AWS_SECRET_ACCESS_KEY'] != "" and os.environ['AWS_ACCESS_KEY_ID'] != "" def prepare(self): self.bucket_name = "aptly-sys-test-" + str(uuid.uuid1()) diff --git a/system/t04_mirror/create.py b/system/t04_mirror/create.py index a2cdbf74..351bb20e 100644 --- a/system/t04_mirror/create.py +++ b/system/t04_mirror/create.py @@ -416,6 +416,7 @@ class CreateMirror31Test(BaseTest): runCmd = "aptly mirror create --keyring=aptlytest-gpg1.gpg mirror11 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch" configOverride = {"gpgProvider": "internal", "max-tries": 1} fixtureGpg = True + faketime = True def outputMatchPrepare(self, s): return re.sub(r'Signature made .* using', '', s)