diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45e9220..71674b38 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,7 @@ jobs: AZURE_STORAGE_ACCOUNT: "devstoreaccount1" AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" run: | + sudo mkdir -p /srv ; sudo chown runner /srv COVERAGE_DIR=${{ runner.temp }} make - name: Merge code coverage diff --git a/Makefile b/Makefile index fe171292..7c525fa9 100644 --- a/Makefile +++ b/Makefile @@ -12,16 +12,13 @@ COVERAGE_DIR?=$(shell mktemp -d) # Uncomment to update test outputs # CAPTURE := "--capture" -all: modules test bench check system-test system-test-etcd +all: modules test bench check system-test # Self-documenting Makefile # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html help: ## Print this help @grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -prepare: - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin $(GOLANGCI_LINT_VERSION) - modules: go mod download go mod verify @@ -53,9 +50,14 @@ endif system-test: install system/env ifeq ($(RUN_LONG_TESTS), yes) go generate + test -d /srv/etcd || system/t13_etcd/install-etcd.sh + system/t13_etcd/start-etcd.sh & go test -v -coverpkg="./..." -c -tags testruncli + kill `cat /tmp/etcd.pid` + if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi + cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) endif @@ -69,16 +71,11 @@ docker-test: install $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST) test: + test -d /srv/etcd || system/t13_etcd/install-etcd.sh + system/t13_etcd/start-etcd.sh & + echo Running go test go test -v ./... -gocheck.v=true -coverprofile=unit.out - -system-test-etcd: install system/env -ifeq ($(RUN_LONG_TESTS), yes) - if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi - if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi - # TODO: maybe we can skip imgrading levledb data to etcd - PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && $(PYTHON) system/leveldb2etcd.py --datadir ~/aptly-fixture-db - PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_DATABASE_TYPE="etcd" APTLY_DATABASE_URL="127.0.0.1:2379" APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) -endif + kill `cat /tmp/etcd.pid` bench: go test -v ./deb -run=nothing -bench=. -benchmem @@ -113,7 +110,7 @@ docker-build-system-tests: ## Build system-test docker image docker build -f system/Dockerfile . -t aptly-system-test docker-unit-tests: ## Run unit tests in docker container - docker run -it --rm -v ${PWD}:/app aptly-system-test go test -v ./... -gocheck.v=true + docker run -it --rm -v ${PWD}:/app aptly-system-test /app/system/run-unit-tests docker-system-tests: ## Run system tests in docker container (add TEST=t04_mirror to run only specific tests) docker run -it --rm -v ${PWD}:/app aptly-system-test /app/system/run-system-tests $(TEST) diff --git a/system/Dockerfile b/system/Dockerfile index 9988f42a..39e82d8b 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -1,6 +1,6 @@ FROM debian:bookworm-slim -RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ && \ +RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel && \ apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly @@ -16,6 +16,7 @@ RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db ADD . /src RUN chown aptly -R /src RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy" +RUN /src/system/t13_etcd/install-etcd.sh RUN rm -rf /src CMD /app/system/run-system-tests diff --git a/system/leveldb2etcd.py b/system/leveldb2etcd.py index d7c18fdd..65788b12 100644 --- a/system/leveldb2etcd.py +++ b/system/leveldb2etcd.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import leveldb +import plyvel import etcd3 import argparse from termcolor import cprint @@ -13,15 +13,15 @@ if __name__ == "__main__": args = parser.parse_args() - ldb = leveldb.LevelDB(args.datadir) + ldb = plyvel.DB(args.datadir) etcd = etcd3.client(args.etcdaddr, args.etcdport) - for key, value in ldb.RangeIter(): + for key, value in ldb: try: - keystr = str(bytes(key)) - valuestr = str(bytes(value)) + keystr = key + valuestr = value etcd.put(keystr, valuestr) # cprint("key: "+keystr+", value: "+valuestr+"put success!\n", 'green') except Exception as e: - cprint("key: " + keystr + ", value: " + valuestr + "put err: " + str(e) + "\n", 'red') + cprint("key: " + str(keystr) + ", value: " + str(valuestr) + "put err: " + str(e) + "\n", 'red') exit(1) diff --git a/system/lib.py b/system/lib.py index d4ce5e25..128b0be3 100644 --- a/system/lib.py +++ b/system/lib.py @@ -129,44 +129,14 @@ class BaseTest(object): requiresDot = False sortOutput = False debugOutput = False + EtcdServer = None aptlyDir = ".aptly" aptlyConfigFile = ".aptly.conf" expectedCode = 0 - databaseType = os.environ.get("APTLY_DATABASE_TYPE") - databaseUrl = os.environ.get("APTLY_DATABASE_URL") - if databaseType is None: - databaseType = "" - if databaseUrl is None: - databaseUrl = "" + databaseType = "" + databaseUrl = "" - databaseBackend = { - "type": databaseType, - "url": databaseUrl, - } - - configFile = { - "rootDir": f"{os.environ['HOME']}/{aptlyDir}", - "downloadConcurrency": 4, - "downloadSpeedLimit": 0, - "downloadRetries": 5, - "databaseOpenAttempts": 10, - "architectures": [], - "dependencyFollowSuggests": False, - "dependencyFollowRecommends": False, - "dependencyFollowAllVariants": False, - "dependencyFollowSource": False, - "gpgDisableVerify": False, - "gpgDisableSign": False, - "ppaDistributorID": "ubuntu", - "ppaCodename": "", - "enableMetricsEndpoint": True, - "logLevel": "debug", - "logFormat": "default", - "serveInAPIMode": True, - "databaseEtcd": databaseEtcd, - "databaseBackend": databaseBackend, - } configOverride = {} environmentOverride = {} @@ -209,7 +179,32 @@ class BaseTest(object): os.environ["HOME"], ".gnupg", "aptlytest.gpg")) def prepare_default_config(self): - cfg = self.configFile.copy() + databaseBackend = { + "type": self.databaseType, + "url": self.databaseUrl, + } + + cfg = { + "rootDir": f"{os.environ['HOME']}/{self.aptlyDir}", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "downloadRetries": 5, + "databaseOpenAttempts": 10, + "architectures": [], + "dependencyFollowSuggests": False, + "dependencyFollowRecommends": False, + "dependencyFollowAllVariants": False, + "dependencyFollowSource": False, + "gpgDisableVerify": False, + "gpgDisableSign": False, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "enableMetricsEndpoint": True, + "logLevel": "debug", + "logFormat": "default", + "serveInAPIMode": True, + "databaseBackend": databaseBackend, + } if self.requiresGPG1: cfg["gpgProvider"] = "gpg1" elif self.requiresGPG2: @@ -246,9 +241,27 @@ class BaseTest(object): shutil.copytree(self.fixturePoolDir, os.path.join( os.environ["HOME"], self.aptlyDir, "pool"), ignore=shutil.ignore_patterns(".git")) - if self.fixtureDB: - shutil.copytree(self.fixtureDBDir, os.path.join( - os.environ["HOME"], self.aptlyDir, "db")) + if self.databaseType == "etcd": + if not os.path.exists("/srv/etcd"): + self.run_cmd([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/install-etcd.sh")]) + + if self.fixtureDB and self.databaseType != "etcd": + shutil.copytree(self.fixtureDBDir, os.path.join(os.environ["HOME"], self.aptlyDir, "db")) + + if self.databaseType == "etcd": + if self.EtcdServer: + self.shutdown_etcd() + + # remove existing database + if os.path.exists("/tmp/etcd-data"): + shutil.rmtree("/tmp/etcd-data") + + if self.fixtureDB: + print("import etcd") + self.run_cmd(["/srv/etcd/etcdctl", "--data-dir=/tmp/etcd-data", "snapshot", "restore", os.path.join(os.environ["HOME"], "etcd.db")]) + + print("starting etcd") + self.EtcdServer = self._start_process([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/start-etcd.sh")], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if self.fixtureWebServer: self.webServerUrl = self.start_webserver(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)), @@ -499,7 +512,8 @@ class BaseTest(object): self.prepare_fixture() def teardown(self): - pass + if self.EtcdServer: + self.shutdown_etcd() def start_webserver(self, directory): FileHTTPServerRequestHandler.rootPath = directory @@ -515,10 +529,18 @@ class BaseTest(object): def shutdown(self): if hasattr(self, 'webserver'): self.shutdown_webserver() + if self.EtcdServer: + self.shutdown_etcd() def shutdown_webserver(self): self.webserver.shutdown() + def shutdown_etcd(self): + print("stopping etcd") + self.EtcdServer.terminate() + self.EtcdServer.wait() + self.EtcdServer = None + @classmethod def shutdown_class(cls): pass diff --git a/system/run-unit-tests b/system/run-unit-tests new file mode 100755 index 00000000..80a4022c --- /dev/null +++ b/system/run-unit-tests @@ -0,0 +1,14 @@ +#!/bin/sh + +# cleanup +rm -rf /app/tmp +rm -rf /tmp/aptly* + +mkdir -p /srv + +usermod -u `stat -c %u /app` aptly >/dev/null +chown -R `stat -c %u /app` /var/lib/aptly /srv + +# use same /home/runner dir as in github workflow +chown -R `stat -c %u /app` /home/runner +su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make test" diff --git a/system/run.py b/system/run.py index dcc3f72b..419dd4d1 100755 --- a/system/run.py +++ b/system/run.py @@ -69,7 +69,13 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non orig_stdout.write(f"error importing: {test + '.' + fname}: {exc}\n") continue + testignore = [] + if hasattr(testModule, "TEST_IGNORE"): + testignore = testModule.TEST_IGNORE for name in sorted(dir(testModule), key=natural_key): + if name in testignore: + continue + testout.clear() o = getattr(testModule, name) diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index ad12f307..3ff0ae2a 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -30,6 +30,9 @@ "logLevel": "debug", "logFormat": "default", "serveInAPIMode": true, - "databaseEtcd": "", - "databaseBackend": {} + "databaseBackend": { + "type": "", + "url": "", + "dbPath": "" + } } diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index 24bed7f6..ab8c7de6 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -30,6 +30,9 @@ "logLevel": "debug", "logFormat": "default", "serveInAPIMode": false, - "databaseEtcd": "", - "databaseBackend": {} + "databaseBackend": { + "type": "", + "url": "", + "dbPath": "" + } } diff --git a/system/t06_publish/drop.py b/system/t06_publish/drop.py index 8c4978e7..3c361cc3 100644 --- a/system/t06_publish/drop.py +++ b/system/t06_publish/drop.py @@ -5,6 +5,7 @@ class PublishDrop1Test(BaseTest): """ publish drop: existing snapshot """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -25,6 +26,7 @@ class PublishDrop2Test(BaseTest): """ publish drop: under prefix """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -46,6 +48,7 @@ class PublishDrop3Test(BaseTest): """ publish drop: drop one distribution """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -68,6 +71,7 @@ class PublishDrop4Test(BaseTest): """ publish drop: drop one of components """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -91,6 +95,7 @@ class PublishDrop5Test(BaseTest): """ publish drop: component cleanup """ + requiresGPG2 = True fixtureCmds = [ "aptly repo create local1", "aptly repo create local2", @@ -128,6 +133,7 @@ class PublishDrop7Test(BaseTest): """ publish drop: under prefix with trailing & leading slashes """ + requiresGPG2 = True fixtureDB = True fixturePool = True fixtureCmds = [ @@ -149,6 +155,7 @@ class PublishDrop8Test(BaseTest): """ publish drop: skip component cleanup """ + requiresGPG2 = True fixtureCmds = [ "aptly repo create local1", "aptly repo create local2", @@ -178,6 +185,7 @@ class PublishDrop9Test(BaseTest): """ publish drop: component cleanup after first cleanup skipped """ + requiresGPG2 = True fixtureCmds = [ "aptly repo create local1", "aptly repo create local2", diff --git a/system/t13_etcd/PublishDrop1TestEtcd_gold b/system/t13_etcd/PublishDrop1TestEtcd_gold new file mode 120000 index 00000000..e96691b2 --- /dev/null +++ b/system/t13_etcd/PublishDrop1TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop1Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop2TestEtcd_gold b/system/t13_etcd/PublishDrop2TestEtcd_gold new file mode 120000 index 00000000..e507a299 --- /dev/null +++ b/system/t13_etcd/PublishDrop2TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop2Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop3TestEtcd_gold b/system/t13_etcd/PublishDrop3TestEtcd_gold new file mode 120000 index 00000000..b371d5c0 --- /dev/null +++ b/system/t13_etcd/PublishDrop3TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop3Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop4TestEtcd_gold b/system/t13_etcd/PublishDrop4TestEtcd_gold new file mode 120000 index 00000000..34a46620 --- /dev/null +++ b/system/t13_etcd/PublishDrop4TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop4Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop5TestEtcd_gold b/system/t13_etcd/PublishDrop5TestEtcd_gold new file mode 120000 index 00000000..3ef73804 --- /dev/null +++ b/system/t13_etcd/PublishDrop5TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop5Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop6TestEtcd_gold b/system/t13_etcd/PublishDrop6TestEtcd_gold new file mode 120000 index 00000000..471ba43d --- /dev/null +++ b/system/t13_etcd/PublishDrop6TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop6Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop7TestEtcd_gold b/system/t13_etcd/PublishDrop7TestEtcd_gold new file mode 120000 index 00000000..a22742fc --- /dev/null +++ b/system/t13_etcd/PublishDrop7TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop7Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop8TestEtcd_gold b/system/t13_etcd/PublishDrop8TestEtcd_gold new file mode 120000 index 00000000..e20efea8 --- /dev/null +++ b/system/t13_etcd/PublishDrop8TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop8Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishDrop9TestEtcd_gold b/system/t13_etcd/PublishDrop9TestEtcd_gold new file mode 120000 index 00000000..89842314 --- /dev/null +++ b/system/t13_etcd/PublishDrop9TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishDrop9Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishList1TestEtcd_gold b/system/t13_etcd/PublishList1TestEtcd_gold new file mode 120000 index 00000000..c9ff7952 --- /dev/null +++ b/system/t13_etcd/PublishList1TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishList1Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishList2TestEtcd_gold b/system/t13_etcd/PublishList2TestEtcd_gold new file mode 120000 index 00000000..383fe5b8 --- /dev/null +++ b/system/t13_etcd/PublishList2TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishList2Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishList3TestEtcd_gold b/system/t13_etcd/PublishList3TestEtcd_gold new file mode 120000 index 00000000..09ac06cf --- /dev/null +++ b/system/t13_etcd/PublishList3TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishList3Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishList4TestEtcd_gold b/system/t13_etcd/PublishList4TestEtcd_gold new file mode 120000 index 00000000..f5b26f53 --- /dev/null +++ b/system/t13_etcd/PublishList4TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishList4Test_gold \ No newline at end of file diff --git a/system/t13_etcd/PublishList5TestEtcd_gold b/system/t13_etcd/PublishList5TestEtcd_gold new file mode 120000 index 00000000..eb94ffde --- /dev/null +++ b/system/t13_etcd/PublishList5TestEtcd_gold @@ -0,0 +1 @@ +../t06_publish/PublishList5Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror10TestEtcd_gold b/system/t13_etcd/UpdateMirror10TestEtcd_gold new file mode 120000 index 00000000..efa730fa --- /dev/null +++ b/system/t13_etcd/UpdateMirror10TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror10Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror11TestEtcd_gold b/system/t13_etcd/UpdateMirror11TestEtcd_gold new file mode 120000 index 00000000..9fb5a6a7 --- /dev/null +++ b/system/t13_etcd/UpdateMirror11TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror11Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror12TestEtcd_gold b/system/t13_etcd/UpdateMirror12TestEtcd_gold new file mode 120000 index 00000000..653d9a72 --- /dev/null +++ b/system/t13_etcd/UpdateMirror12TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror12Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror13TestEtcd_gold b/system/t13_etcd/UpdateMirror13TestEtcd_gold new file mode 120000 index 00000000..e5f8e8cf --- /dev/null +++ b/system/t13_etcd/UpdateMirror13TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror13Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror14TestEtcd_gold b/system/t13_etcd/UpdateMirror14TestEtcd_gold new file mode 120000 index 00000000..0f7bf746 --- /dev/null +++ b/system/t13_etcd/UpdateMirror14TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror14Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror15TestEtcd_gold b/system/t13_etcd/UpdateMirror15TestEtcd_gold new file mode 120000 index 00000000..bae34dff --- /dev/null +++ b/system/t13_etcd/UpdateMirror15TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror15Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror16TestEtcd_gold b/system/t13_etcd/UpdateMirror16TestEtcd_gold new file mode 120000 index 00000000..edbe7f22 --- /dev/null +++ b/system/t13_etcd/UpdateMirror16TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror16Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror17TestEtcd_gold b/system/t13_etcd/UpdateMirror17TestEtcd_gold new file mode 120000 index 00000000..eb1abbe6 --- /dev/null +++ b/system/t13_etcd/UpdateMirror17TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror17Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror18TestEtcd_gold b/system/t13_etcd/UpdateMirror18TestEtcd_gold new file mode 120000 index 00000000..c0333d25 --- /dev/null +++ b/system/t13_etcd/UpdateMirror18TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror18Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror19TestEtcd_gold b/system/t13_etcd/UpdateMirror19TestEtcd_gold new file mode 120000 index 00000000..a178fe7b --- /dev/null +++ b/system/t13_etcd/UpdateMirror19TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror19Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror1TestEtcd_gold b/system/t13_etcd/UpdateMirror1TestEtcd_gold new file mode 120000 index 00000000..cc5035be --- /dev/null +++ b/system/t13_etcd/UpdateMirror1TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror1Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror20TestEtcd_gold b/system/t13_etcd/UpdateMirror20TestEtcd_gold new file mode 120000 index 00000000..444b11de --- /dev/null +++ b/system/t13_etcd/UpdateMirror20TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror20Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror21TestEtcd_gold b/system/t13_etcd/UpdateMirror21TestEtcd_gold new file mode 120000 index 00000000..bff753d0 --- /dev/null +++ b/system/t13_etcd/UpdateMirror21TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror21Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror22TestEtcd_gold b/system/t13_etcd/UpdateMirror22TestEtcd_gold new file mode 120000 index 00000000..54697e8c --- /dev/null +++ b/system/t13_etcd/UpdateMirror22TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror22Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror23TestEtcd_gold b/system/t13_etcd/UpdateMirror23TestEtcd_gold new file mode 120000 index 00000000..649ac10f --- /dev/null +++ b/system/t13_etcd/UpdateMirror23TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror23Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror24TestEtcd_gold b/system/t13_etcd/UpdateMirror24TestEtcd_gold new file mode 120000 index 00000000..31721619 --- /dev/null +++ b/system/t13_etcd/UpdateMirror24TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror24Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror25TestEtcd_gold b/system/t13_etcd/UpdateMirror25TestEtcd_gold new file mode 120000 index 00000000..a3a057cf --- /dev/null +++ b/system/t13_etcd/UpdateMirror25TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror25Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror2TestEtcd_gold b/system/t13_etcd/UpdateMirror2TestEtcd_gold new file mode 120000 index 00000000..9bf2cf84 --- /dev/null +++ b/system/t13_etcd/UpdateMirror2TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror2Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror3TestEtcd_gold b/system/t13_etcd/UpdateMirror3TestEtcd_gold new file mode 120000 index 00000000..3a18595b --- /dev/null +++ b/system/t13_etcd/UpdateMirror3TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror3Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror4TestEtcd_gold b/system/t13_etcd/UpdateMirror4TestEtcd_gold new file mode 120000 index 00000000..33efda8a --- /dev/null +++ b/system/t13_etcd/UpdateMirror4TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror4Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror5TestEtcd_gold b/system/t13_etcd/UpdateMirror5TestEtcd_gold new file mode 120000 index 00000000..3dd08aed --- /dev/null +++ b/system/t13_etcd/UpdateMirror5TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror5Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror6TestEtcd_gold b/system/t13_etcd/UpdateMirror6TestEtcd_gold new file mode 120000 index 00000000..3ddffa3a --- /dev/null +++ b/system/t13_etcd/UpdateMirror6TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror6Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror7TestEtcd_gold b/system/t13_etcd/UpdateMirror7TestEtcd_gold new file mode 120000 index 00000000..283a8c0c --- /dev/null +++ b/system/t13_etcd/UpdateMirror7TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror7Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror8TestEtcd_gold b/system/t13_etcd/UpdateMirror8TestEtcd_gold new file mode 120000 index 00000000..2cb66953 --- /dev/null +++ b/system/t13_etcd/UpdateMirror8TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror8Test_gold \ No newline at end of file diff --git a/system/t13_etcd/UpdateMirror9TestEtcd_gold b/system/t13_etcd/UpdateMirror9TestEtcd_gold new file mode 120000 index 00000000..4105a0d6 --- /dev/null +++ b/system/t13_etcd/UpdateMirror9TestEtcd_gold @@ -0,0 +1 @@ +../t04_mirror/UpdateMirror9Test_gold \ No newline at end of file diff --git a/system/t13_etcd/__init__.py b/system/t13_etcd/__init__.py new file mode 100644 index 00000000..d3a06023 --- /dev/null +++ b/system/t13_etcd/__init__.py @@ -0,0 +1,3 @@ +""" +Testing aptly with etcd DB +""" diff --git a/system/t13_etcd/install-etcd.sh b/system/t13_etcd/install-etcd.sh new file mode 100755 index 00000000..163a3f85 --- /dev/null +++ b/system/t13_etcd/install-etcd.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +# etcd test env +ETCD_VER=v3.5.2 +DOWNLOAD_URL=https://storage.googleapis.com/etcd + +if [ ! -e /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ]; then + curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz +fi + +mkdir /srv/etcd +tar xf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /srv/etcd --strip-components=1 diff --git a/system/t13_etcd/mirror_update.py b/system/t13_etcd/mirror_update.py new file mode 100644 index 00000000..d03b2c4d --- /dev/null +++ b/system/t13_etcd/mirror_update.py @@ -0,0 +1,234 @@ +# reuse existing tests: +from t04_mirror.update import UpdateMirror1Test, \ + UpdateMirror2Test, \ + UpdateMirror3Test, \ + UpdateMirror4Test, \ + UpdateMirror5Test, \ + UpdateMirror6Test, \ + UpdateMirror7Test, \ + UpdateMirror8Test, \ + UpdateMirror9Test, \ + UpdateMirror10Test, \ + UpdateMirror11Test, \ + UpdateMirror12Test, \ + UpdateMirror13Test, \ + UpdateMirror14Test, \ + UpdateMirror17Test, \ + UpdateMirror18Test, \ + UpdateMirror19Test, \ + UpdateMirror20Test, \ + UpdateMirror21Test, \ + UpdateMirror22Test, \ + UpdateMirror23Test, \ + UpdateMirror24Test, \ + UpdateMirror25Test + + +TEST_IGNORE = ["UpdateMirror1Test", + "UpdateMirror2Test", + "UpdateMirror3Test", + "UpdateMirror4Test", + "UpdateMirror5Test", + "UpdateMirror6Test", + "UpdateMirror7Test", + "UpdateMirror8Test", + "UpdateMirror9Test", + "UpdateMirror10Test", + "UpdateMirror11Test", + "UpdateMirror12Test", + "UpdateMirror13Test", + "UpdateMirror14Test", + "UpdateMirror17Test", + "UpdateMirror18Test", + "UpdateMirror19Test", + "UpdateMirror20Test", + "UpdateMirror21Test", + "UpdateMirror22Test", + "UpdateMirror23Test", + "UpdateMirror24Test", + "UpdateMirror25Test" + ] + + +class UpdateMirror1TestEtcd(UpdateMirror1Test): + """ + update mirrors: regular update + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror2TestEtcd(UpdateMirror2Test): + """ + update mirrors: no such repo + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror3TestEtcd(UpdateMirror3Test): + """ + update mirrors: wrong checksum in release file + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror4TestEtcd(UpdateMirror4Test): + """ + update mirrors: wrong checksum in release file, but ignore + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror5TestEtcd(UpdateMirror5Test): + """ + update mirrors: wrong checksum in package + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror6TestEtcd(UpdateMirror6Test): + """ + update mirrors: wrong checksum in package, but ignore + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror7TestEtcd(UpdateMirror7Test): + """ + update mirrors: flat repository + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror8TestEtcd(UpdateMirror8Test): + """ + update mirrors: with sources (already in pool) + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror9TestEtcd(UpdateMirror9Test): + """ + update mirrors: flat repository + sources + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror10TestEtcd(UpdateMirror10Test): + """ + update mirrors: filtered + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror11TestEtcd(UpdateMirror11Test): + """ + update mirrors: update over FTP + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror12TestEtcd(UpdateMirror12Test): + """ + update mirrors: update with udebs + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror13TestEtcd(UpdateMirror13Test): + """ + update mirrors: regular update with --skip-existing-packages option + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror14TestEtcd(UpdateMirror14Test): + """ + update mirrors: regular update with --skip-existing-packages option + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror17TestEtcd(UpdateMirror17Test): + """ + update mirrors: update for mirror but with file in pool on legacy MD5 location + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror18TestEtcd(UpdateMirror18Test): + """ + update mirrors: update for mirror but with file in pool on legacy MD5 location and disabled legacy path support + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror19TestEtcd(UpdateMirror19Test): + """ + update mirrors: correct matching of Release checksums + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror20TestEtcd(UpdateMirror20Test): + """ + update mirrors: flat repository (internal GPG implementation) + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror21TestEtcd(UpdateMirror21Test): + """ + update mirrors: correct matching of Release checksums (internal pgp implementation) + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror22TestEtcd(UpdateMirror22Test): + """ + update mirrors: SHA512 checksums only + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror23TestEtcd(UpdateMirror23Test): + """ + update mirrors: update with installer + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror24TestEtcd(UpdateMirror24Test): + """ + update mirrors: update with installer with separate gpg file + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class UpdateMirror25TestEtcd(UpdateMirror25Test): + """ + update mirrors: mirror with / in distribution + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" diff --git a/system/t13_etcd/publish_drop.py b/system/t13_etcd/publish_drop.py new file mode 100644 index 00000000..8f9f60fc --- /dev/null +++ b/system/t13_etcd/publish_drop.py @@ -0,0 +1,92 @@ +# reuse existing tests: +from t06_publish.drop import PublishDrop1Test, \ + PublishDrop2Test, \ + PublishDrop3Test, \ + PublishDrop4Test, \ + PublishDrop5Test, \ + PublishDrop6Test, \ + PublishDrop7Test, \ + PublishDrop8Test, \ + PublishDrop9Test + +TEST_IGNORE = ["PublishDrop1Test", + "PublishDrop2Test", + "PublishDrop3Test", + "PublishDrop4Test", + "PublishDrop5Test", + "PublishDrop6Test", + "PublishDrop7Test", + "PublishDrop8Test", + "PublishDrop9Test"] + + +class PublishDrop1TestEtcd(PublishDrop1Test): + """ + publish drop: existing snapshot + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop2TestEtcd(PublishDrop2Test): + """ + publish drop: under prefix + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop3TestEtcd(PublishDrop3Test): + """ + publish drop: drop one distribution + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop4TestEtcd(PublishDrop4Test): + """ + publish drop: drop one of components + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop5TestEtcd(PublishDrop5Test): + """ + publish drop: component cleanup + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop6TestEtcd(PublishDrop6Test): + """ + publish drop: no publish + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop7TestEtcd(PublishDrop7Test): + """ + publish drop: under prefix with trailing & leading slashes + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop8TestEtcd(PublishDrop8Test): + """ + publish drop: skip component cleanup + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishDrop9TestEtcd(PublishDrop9Test): + """ + publish drop: component cleanup after first cleanup skipped + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" diff --git a/system/t13_etcd/publish_list.py b/system/t13_etcd/publish_list.py new file mode 100644 index 00000000..8825d05e --- /dev/null +++ b/system/t13_etcd/publish_list.py @@ -0,0 +1,48 @@ +# reuse existing tests: +from t06_publish.list import PublishList1Test, \ + PublishList2Test, \ + PublishList3Test, \ + PublishList4Test, \ + PublishList5Test + +TEST_IGNORE = ["PublishList1Test", "PublishList2Test", "PublishList3Test", "PublishList4Test", "PublishList5Test"] + + +class PublishList1TestEtcd(PublishList1Test): + """ + publish list: empty list + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishList2TestEtcd(PublishList2Test): + """ + publish list: several repos list + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishList3TestEtcd(PublishList3Test): + """ + publish list: several repos list, raw + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishList4TestEtcd(PublishList4Test): + """ + publish list json: empty list + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" + + +class PublishList5TestEtcd(PublishList5Test): + """ + publish list json: several repos list + """ + databaseType = "etcd" + databaseUrl = "127.0.0.1:2379" diff --git a/system/t13_etcd/start-etcd.sh b/system/t13_etcd/start-etcd.sh new file mode 100755 index 00000000..ae6c0a45 --- /dev/null +++ b/system/t13_etcd/start-etcd.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +if [ -e /tmp/etcd.pid ]; then + echo etcd already running, killing.. + etcdpid=`cat /tmp/etcd.pid` + kill $etcdpid + sleep 2 +fi + +finish() +{ + if [ -n "$etcdpid" ]; then + echo terminating etcd + kill $etcdpid + fi +} +trap finish INT + +/srv/etcd/etcd --max-request-bytes '1073741824' --data-dir /tmp/etcd-data & +echo $! > /tmp/etcd.pid +etcdpid=`cat /tmp/etcd.pid` +wait $etcdpid +echo etcd terminated diff --git a/system/t13_etcd/test_release b/system/t13_etcd/test_release new file mode 120000 index 00000000..e8d20bcb --- /dev/null +++ b/system/t13_etcd/test_release @@ -0,0 +1 @@ +../t04_mirror/test_release \ No newline at end of file diff --git a/system/t13_etcd/test_release2 b/system/t13_etcd/test_release2 new file mode 120000 index 00000000..a77f3071 --- /dev/null +++ b/system/t13_etcd/test_release2 @@ -0,0 +1 @@ +../t04_mirror/test_release2 \ No newline at end of file