etcd: implement separate system tests

- add t13_etcd test directory
- etcd will be started for the unit tests and each system test
- etcd will load fixture DB export if requested by the test
- existing tests are reused for etcd testing

.github/workflows/ci.yml

@@ -75,6 +75,7 @@ jobs:
           AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
           AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
         run: |
+          sudo mkdir -p /srv ; sudo chown runner /srv
           COVERAGE_DIR=${{ runner.temp }} make
 
       - name: Merge code coverage

Makefile

@@ -12,16 +12,13 @@ COVERAGE_DIR?=$(shell mktemp -d)
 # Uncomment to update test outputs
 # CAPTURE := "--capture"
 
-all: modules test bench check system-test system-test-etcd
+all: modules test bench check system-test
 
 # Self-documenting Makefile
 # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 help:  ## Print this help
 	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-prepare:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
-
 modules:
 	go mod download
 	go mod verify
@@ -53,9 +50,14 @@ endif
 system-test: install system/env
 ifeq ($(RUN_LONG_TESTS), yes)
 	go generate
+	test -d /srv/etcd || system/t13_etcd/install-etcd.sh
+	system/t13_etcd/start-etcd.sh &
 	go test -v -coverpkg="./..." -c -tags testruncli
+	kill `cat /tmp/etcd.pid`
+
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db
 	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
 endif
 
@@ -69,16 +71,11 @@ docker-test: install
 	$(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
 test:
+	test -d /srv/etcd || system/t13_etcd/install-etcd.sh
+	system/t13_etcd/start-etcd.sh &
+	echo Running go test
 	go test -v ./... -gocheck.v=true -coverprofile=unit.out
-
+	kill `cat /tmp/etcd.pid`
-system-test-etcd: install system/env
-ifeq ($(RUN_LONG_TESTS), yes)
-	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
-	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
-	# TODO: maybe we can skip imgrading levledb data to etcd
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && $(PYTHON) system/leveldb2etcd.py --datadir ~/aptly-fixture-db
-	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_DATABASE_TYPE="etcd" APTLY_DATABASE_URL="127.0.0.1:2379" APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
-endif
 
 bench:
 	go test -v ./deb -run=nothing -bench=. -benchmem
@@ -113,7 +110,7 @@ docker-build-system-tests:  ## Build system-test docker image
 	docker build -f system/Dockerfile . -t aptly-system-test
 
 docker-unit-tests:  ## Run unit tests in docker container
-	docker run -it --rm -v ${PWD}:/app aptly-system-test go test -v ./... -gocheck.v=true
+	docker run -it --rm -v ${PWD}:/app aptly-system-test /app/system/run-unit-tests
 
 docker-system-tests:  ## Run system tests in docker container (add TEST=t04_mirror to run only specific tests)
 	docker run -it --rm -v ${PWD}:/app aptly-system-test /app/system/run-system-tests $(TEST)

system/Dockerfile

@@ -1,6 +1,6 @@
 FROM debian:bookworm-slim
 
-RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ && \
+RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg apg bzip2 xz-utils ca-certificates golang golang-go golang-doc golang-src make git python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage g++ python3-etcd3 python3-plyvel && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
@@ -16,6 +16,7 @@ RUN cd /home/runner; curl -O http://repo.aptly.info/system-tests/etcd.db
 ADD . /src
 RUN chown aptly -R /src
 RUN cd /src; su aptly -c "HOME=/home/runner go mod tidy"
+RUN /src/system/t13_etcd/install-etcd.sh
 RUN rm -rf /src
 
 CMD /app/system/run-system-tests

system/leveldb2etcd.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import leveldb
+import plyvel
 import etcd3
 import argparse
 from termcolor import cprint
@@ -13,15 +13,15 @@ if __name__ == "__main__":
 
     args = parser.parse_args()
 
-    ldb = leveldb.LevelDB(args.datadir)
+    ldb = plyvel.DB(args.datadir)
     etcd = etcd3.client(args.etcdaddr, args.etcdport)
 
-    for key, value in ldb.RangeIter():
+    for key, value in ldb:
         try:
-            keystr = str(bytes(key))
+            keystr = key
-            valuestr = str(bytes(value))
+            valuestr = value
             etcd.put(keystr, valuestr)
             # cprint("key: "+keystr+", value: "+valuestr+"put success!\n", 'green')
         except Exception as e:
-            cprint("key: " + keystr + ", value: " + valuestr + "put err: " + str(e) + "\n", 'red')
+            cprint("key: " + str(keystr) + ", value: " + str(valuestr) + "put err: " + str(e) + "\n", 'red')
             exit(1)

system/lib.py

@@ -129,44 +129,14 @@ class BaseTest(object):
     requiresDot = False
     sortOutput = False
     debugOutput = False
+    EtcdServer = None
 
     aptlyDir = ".aptly"
     aptlyConfigFile = ".aptly.conf"
     expectedCode = 0
-    databaseType = os.environ.get("APTLY_DATABASE_TYPE")
+    databaseType = ""
-    databaseUrl = os.environ.get("APTLY_DATABASE_URL")
+    databaseUrl = ""
-    if databaseType is None:
-        databaseType = ""
-    if databaseUrl is None:
-        databaseUrl = ""
 
-    databaseBackend = {
-        "type": databaseType,
-        "url": databaseUrl,
-    }
-
-    configFile = {
-        "rootDir": f"{os.environ['HOME']}/{aptlyDir}",
-        "downloadConcurrency": 4,
-        "downloadSpeedLimit": 0,
-        "downloadRetries": 5,
-        "databaseOpenAttempts": 10,
-        "architectures": [],
-        "dependencyFollowSuggests": False,
-        "dependencyFollowRecommends": False,
-        "dependencyFollowAllVariants": False,
-        "dependencyFollowSource": False,
-        "gpgDisableVerify": False,
-        "gpgDisableSign": False,
-        "ppaDistributorID": "ubuntu",
-        "ppaCodename": "",
-        "enableMetricsEndpoint": True,
-        "logLevel": "debug",
-        "logFormat": "default",
-        "serveInAPIMode": True,
-        "databaseEtcd": databaseEtcd,
-        "databaseBackend": databaseBackend,
-    }
     configOverride = {}
     environmentOverride = {}
 
@@ -209,7 +179,32 @@ class BaseTest(object):
                 os.environ["HOME"], ".gnupg", "aptlytest.gpg"))
 
     def prepare_default_config(self):
-        cfg = self.configFile.copy()
+        databaseBackend = {
+            "type": self.databaseType,
+            "url": self.databaseUrl,
+        }
+
+        cfg = {
+            "rootDir": f"{os.environ['HOME']}/{self.aptlyDir}",
+            "downloadConcurrency": 4,
+            "downloadSpeedLimit": 0,
+            "downloadRetries": 5,
+            "databaseOpenAttempts": 10,
+            "architectures": [],
+            "dependencyFollowSuggests": False,
+            "dependencyFollowRecommends": False,
+            "dependencyFollowAllVariants": False,
+            "dependencyFollowSource": False,
+            "gpgDisableVerify": False,
+            "gpgDisableSign": False,
+            "ppaDistributorID": "ubuntu",
+            "ppaCodename": "",
+            "enableMetricsEndpoint": True,
+            "logLevel": "debug",
+            "logFormat": "default",
+            "serveInAPIMode": True,
+            "databaseBackend": databaseBackend,
+        }
         if self.requiresGPG1:
             cfg["gpgProvider"] = "gpg1"
         elif self.requiresGPG2:
@@ -246,9 +241,27 @@ class BaseTest(object):
             shutil.copytree(self.fixturePoolDir, os.path.join(
                 os.environ["HOME"], self.aptlyDir, "pool"), ignore=shutil.ignore_patterns(".git"))
 
-        if self.fixtureDB:
+        if self.databaseType == "etcd":
-            shutil.copytree(self.fixtureDBDir, os.path.join(
+            if not os.path.exists("/srv/etcd"):
-                os.environ["HOME"], self.aptlyDir, "db"))
+                self.run_cmd([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/install-etcd.sh")])
+
+        if self.fixtureDB and self.databaseType != "etcd":
+            shutil.copytree(self.fixtureDBDir, os.path.join(os.environ["HOME"], self.aptlyDir, "db"))
+
+        if self.databaseType == "etcd":
+            if self.EtcdServer:
+                self.shutdown_etcd()
+
+            # remove existing database
+            if os.path.exists("/tmp/etcd-data"):
+                shutil.rmtree("/tmp/etcd-data")
+
+            if self.fixtureDB:
+                print("import etcd")
+                self.run_cmd(["/srv/etcd/etcdctl", "--data-dir=/tmp/etcd-data", "snapshot", "restore", os.path.join(os.environ["HOME"], "etcd.db")])
+
+            print("starting etcd")
+            self.EtcdServer = self._start_process([os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "t13_etcd/start-etcd.sh")], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
         if self.fixtureWebServer:
             self.webServerUrl = self.start_webserver(os.path.join(os.path.dirname(inspect.getsourcefile(self.__class__)),
@@ -499,7 +512,8 @@ class BaseTest(object):
         self.prepare_fixture()
 
     def teardown(self):
-        pass
+        if self.EtcdServer:
+            self.shutdown_etcd()
 
     def start_webserver(self, directory):
         FileHTTPServerRequestHandler.rootPath = directory
@@ -515,10 +529,18 @@ class BaseTest(object):
     def shutdown(self):
         if hasattr(self, 'webserver'):
             self.shutdown_webserver()
+        if self.EtcdServer:
+            self.shutdown_etcd()
 
     def shutdown_webserver(self):
         self.webserver.shutdown()
 
+    def shutdown_etcd(self):
+        print("stopping etcd")
+        self.EtcdServer.terminate()
+        self.EtcdServer.wait()
+        self.EtcdServer = None
+
     @classmethod
     def shutdown_class(cls):
         pass

system/run-unit-tests

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# cleanup
+rm -rf /app/tmp
+rm -rf /tmp/aptly*
+
+mkdir -p /srv
+
+usermod -u `stat -c %u /app` aptly >/dev/null
+chown -R `stat -c %u /app` /var/lib/aptly /srv
+
+# use same /home/runner dir as in github workflow
+chown -R `stat -c %u /app` /home/runner
+su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make test"

system/run.py

@@ -69,7 +69,13 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                 orig_stdout.write(f"error importing: {test + '.' + fname}: {exc}\n")
                 continue
 
+            testignore = []
+            if hasattr(testModule, "TEST_IGNORE"):
+                testignore = testModule.TEST_IGNORE
             for name in sorted(dir(testModule), key=natural_key):
+                if name in testignore:
+                    continue
+
                 testout.clear()
 
                 o = getattr(testModule, name)

system/t02_config/ConfigShowTest_gold

@@ -30,6 +30,9 @@
     "logLevel": "debug",
     "logFormat": "default",
     "serveInAPIMode": true,
-    "databaseEtcd": "",
+    "databaseBackend": {
-    "databaseBackend": {}
+        "type": "",
+        "url": "",
+        "dbPath": ""
+    }
 }

system/t02_config/CreateConfigTest_gold

@@ -30,6 +30,9 @@
   "logLevel": "debug",
   "logFormat": "default",
   "serveInAPIMode": false,
-  "databaseEtcd": "",
+  "databaseBackend": {
-  "databaseBackend": {}
+    "type": "",
+    "url": "",
+    "dbPath": ""
+  }
 }

system/t06_publish/drop.py

@@ -5,6 +5,7 @@ class PublishDrop1Test(BaseTest):
     """
     publish drop: existing snapshot
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -25,6 +26,7 @@ class PublishDrop2Test(BaseTest):
     """
     publish drop: under prefix
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -46,6 +48,7 @@ class PublishDrop3Test(BaseTest):
     """
     publish drop: drop one distribution
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -68,6 +71,7 @@ class PublishDrop4Test(BaseTest):
     """
     publish drop: drop one of components
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -91,6 +95,7 @@ class PublishDrop5Test(BaseTest):
     """
     publish drop: component cleanup
     """
+    requiresGPG2 = True
     fixtureCmds = [
         "aptly repo create local1",
         "aptly repo create local2",
@@ -128,6 +133,7 @@ class PublishDrop7Test(BaseTest):
     """
     publish drop: under prefix with trailing & leading slashes
     """
+    requiresGPG2 = True
     fixtureDB = True
     fixturePool = True
     fixtureCmds = [
@@ -149,6 +155,7 @@ class PublishDrop8Test(BaseTest):
     """
     publish drop: skip component cleanup
     """
+    requiresGPG2 = True
     fixtureCmds = [
         "aptly repo create local1",
         "aptly repo create local2",
@@ -178,6 +185,7 @@ class PublishDrop9Test(BaseTest):
     """
     publish drop: component cleanup after first cleanup skipped
     """
+    requiresGPG2 = True
     fixtureCmds = [
         "aptly repo create local1",
         "aptly repo create local2",

system/t13_etcd/PublishDrop1TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop1Test_gold

system/t13_etcd/PublishDrop2TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop2Test_gold

system/t13_etcd/PublishDrop3TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop3Test_gold

system/t13_etcd/PublishDrop4TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop4Test_gold

system/t13_etcd/PublishDrop5TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop5Test_gold

system/t13_etcd/PublishDrop6TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop6Test_gold

system/t13_etcd/PublishDrop7TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop7Test_gold

system/t13_etcd/PublishDrop8TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop8Test_gold

system/t13_etcd/PublishDrop9TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishDrop9Test_gold

system/t13_etcd/PublishList1TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishList1Test_gold

system/t13_etcd/PublishList2TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishList2Test_gold

system/t13_etcd/PublishList3TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishList3Test_gold

system/t13_etcd/PublishList4TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishList4Test_gold

system/t13_etcd/PublishList5TestEtcd_gold

@@ -0,0 +1 @@
+../t06_publish/PublishList5Test_gold

system/t13_etcd/UpdateMirror10TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror10Test_gold

system/t13_etcd/UpdateMirror11TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror11Test_gold

system/t13_etcd/UpdateMirror12TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror12Test_gold

system/t13_etcd/UpdateMirror13TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror13Test_gold

system/t13_etcd/UpdateMirror14TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror14Test_gold

system/t13_etcd/UpdateMirror15TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror15Test_gold

system/t13_etcd/UpdateMirror16TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror16Test_gold

system/t13_etcd/UpdateMirror17TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror17Test_gold

system/t13_etcd/UpdateMirror18TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror18Test_gold

system/t13_etcd/UpdateMirror19TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror19Test_gold

system/t13_etcd/UpdateMirror1TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror1Test_gold

system/t13_etcd/UpdateMirror20TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror20Test_gold

system/t13_etcd/UpdateMirror21TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror21Test_gold

system/t13_etcd/UpdateMirror22TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror22Test_gold

system/t13_etcd/UpdateMirror23TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror23Test_gold

system/t13_etcd/UpdateMirror24TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror24Test_gold

system/t13_etcd/UpdateMirror25TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror25Test_gold

system/t13_etcd/UpdateMirror2TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror2Test_gold

system/t13_etcd/UpdateMirror3TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror3Test_gold

system/t13_etcd/UpdateMirror4TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror4Test_gold

system/t13_etcd/UpdateMirror5TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror5Test_gold

system/t13_etcd/UpdateMirror6TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror6Test_gold

system/t13_etcd/UpdateMirror7TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror7Test_gold

system/t13_etcd/UpdateMirror8TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror8Test_gold

system/t13_etcd/UpdateMirror9TestEtcd_gold

@@ -0,0 +1 @@
+../t04_mirror/UpdateMirror9Test_gold

system/t13_etcd/__init__.py

@@ -0,0 +1,3 @@
+"""
+Testing aptly with etcd DB
+"""

system/t13_etcd/install-etcd.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+# etcd test env
+ETCD_VER=v3.5.2
+DOWNLOAD_URL=https://storage.googleapis.com/etcd
+
+if [ ! -e /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ]; then
+    curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+fi
+
+mkdir /srv/etcd
+tar xf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /srv/etcd --strip-components=1

system/t13_etcd/mirror_update.py

@@ -0,0 +1,234 @@
+# reuse existing tests:
+from t04_mirror.update import UpdateMirror1Test, \
+        UpdateMirror2Test, \
+        UpdateMirror3Test, \
+        UpdateMirror4Test, \
+        UpdateMirror5Test, \
+        UpdateMirror6Test, \
+        UpdateMirror7Test, \
+        UpdateMirror8Test, \
+        UpdateMirror9Test, \
+        UpdateMirror10Test, \
+        UpdateMirror11Test, \
+        UpdateMirror12Test, \
+        UpdateMirror13Test, \
+        UpdateMirror14Test, \
+        UpdateMirror17Test, \
+        UpdateMirror18Test, \
+        UpdateMirror19Test, \
+        UpdateMirror20Test, \
+        UpdateMirror21Test, \
+        UpdateMirror22Test, \
+        UpdateMirror23Test, \
+        UpdateMirror24Test, \
+        UpdateMirror25Test
+
+
+TEST_IGNORE = ["UpdateMirror1Test",
+               "UpdateMirror2Test",
+               "UpdateMirror3Test",
+               "UpdateMirror4Test",
+               "UpdateMirror5Test",
+               "UpdateMirror6Test",
+               "UpdateMirror7Test",
+               "UpdateMirror8Test",
+               "UpdateMirror9Test",
+               "UpdateMirror10Test",
+               "UpdateMirror11Test",
+               "UpdateMirror12Test",
+               "UpdateMirror13Test",
+               "UpdateMirror14Test",
+               "UpdateMirror17Test",
+               "UpdateMirror18Test",
+               "UpdateMirror19Test",
+               "UpdateMirror20Test",
+               "UpdateMirror21Test",
+               "UpdateMirror22Test",
+               "UpdateMirror23Test",
+               "UpdateMirror24Test",
+               "UpdateMirror25Test"
+               ]
+
+
+class UpdateMirror1TestEtcd(UpdateMirror1Test):
+    """
+    update mirrors: regular update
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror2TestEtcd(UpdateMirror2Test):
+    """
+    update mirrors: no such repo
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror3TestEtcd(UpdateMirror3Test):
+    """
+    update mirrors: wrong checksum in release file
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror4TestEtcd(UpdateMirror4Test):
+    """
+    update mirrors: wrong checksum in release file, but ignore
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror5TestEtcd(UpdateMirror5Test):
+    """
+    update mirrors: wrong checksum in package
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror6TestEtcd(UpdateMirror6Test):
+    """
+    update mirrors: wrong checksum in package, but ignore
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror7TestEtcd(UpdateMirror7Test):
+    """
+    update mirrors: flat repository
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror8TestEtcd(UpdateMirror8Test):
+    """
+    update mirrors: with sources (already in pool)
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror9TestEtcd(UpdateMirror9Test):
+    """
+    update mirrors: flat repository + sources
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror10TestEtcd(UpdateMirror10Test):
+    """
+    update mirrors: filtered
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror11TestEtcd(UpdateMirror11Test):
+    """
+    update mirrors: update over FTP
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror12TestEtcd(UpdateMirror12Test):
+    """
+    update mirrors: update with udebs
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror13TestEtcd(UpdateMirror13Test):
+    """
+    update mirrors: regular update with --skip-existing-packages option
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror14TestEtcd(UpdateMirror14Test):
+    """
+    update mirrors: regular update with --skip-existing-packages option
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror17TestEtcd(UpdateMirror17Test):
+    """
+    update mirrors: update for mirror but with file in pool on legacy MD5 location
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror18TestEtcd(UpdateMirror18Test):
+    """
+    update mirrors: update for mirror but with file in pool on legacy MD5 location and disabled legacy path support
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror19TestEtcd(UpdateMirror19Test):
+    """
+    update mirrors: correct matching of Release checksums
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror20TestEtcd(UpdateMirror20Test):
+    """
+    update mirrors: flat repository (internal GPG implementation)
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror21TestEtcd(UpdateMirror21Test):
+    """
+    update mirrors: correct matching of Release checksums (internal pgp implementation)
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror22TestEtcd(UpdateMirror22Test):
+    """
+    update mirrors: SHA512 checksums only
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror23TestEtcd(UpdateMirror23Test):
+    """
+    update mirrors: update with installer
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror24TestEtcd(UpdateMirror24Test):
+    """
+    update mirrors: update with installer with separate gpg file
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class UpdateMirror25TestEtcd(UpdateMirror25Test):
+    """
+    update mirrors: mirror with / in distribution
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"

system/t13_etcd/publish_drop.py

@@ -0,0 +1,92 @@
+# reuse existing tests:
+from t06_publish.drop import PublishDrop1Test, \
+                             PublishDrop2Test, \
+                             PublishDrop3Test, \
+                             PublishDrop4Test, \
+                             PublishDrop5Test, \
+                             PublishDrop6Test, \
+                             PublishDrop7Test, \
+                             PublishDrop8Test, \
+                             PublishDrop9Test
+
+TEST_IGNORE = ["PublishDrop1Test",
+               "PublishDrop2Test",
+               "PublishDrop3Test",
+               "PublishDrop4Test",
+               "PublishDrop5Test",
+               "PublishDrop6Test",
+               "PublishDrop7Test",
+               "PublishDrop8Test",
+               "PublishDrop9Test"]
+
+
+class PublishDrop1TestEtcd(PublishDrop1Test):
+    """
+    publish drop: existing snapshot
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop2TestEtcd(PublishDrop2Test):
+    """
+    publish drop: under prefix
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop3TestEtcd(PublishDrop3Test):
+    """
+    publish drop: drop one distribution
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop4TestEtcd(PublishDrop4Test):
+    """
+    publish drop: drop one of components
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop5TestEtcd(PublishDrop5Test):
+    """
+    publish drop: component cleanup
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop6TestEtcd(PublishDrop6Test):
+    """
+    publish drop: no publish
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop7TestEtcd(PublishDrop7Test):
+    """
+    publish drop: under prefix with trailing & leading slashes
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop8TestEtcd(PublishDrop8Test):
+    """
+    publish drop: skip component cleanup
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishDrop9TestEtcd(PublishDrop9Test):
+    """
+    publish drop: component cleanup after first cleanup skipped
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"

system/t13_etcd/publish_list.py

@@ -0,0 +1,48 @@
+# reuse existing tests:
+from t06_publish.list import PublishList1Test, \
+                             PublishList2Test, \
+                             PublishList3Test, \
+                             PublishList4Test, \
+                             PublishList5Test
+
+TEST_IGNORE = ["PublishList1Test", "PublishList2Test", "PublishList3Test", "PublishList4Test", "PublishList5Test"]
+
+
+class PublishList1TestEtcd(PublishList1Test):
+    """
+    publish list: empty list
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishList2TestEtcd(PublishList2Test):
+    """
+    publish list: several repos list
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishList3TestEtcd(PublishList3Test):
+    """
+    publish list: several repos list, raw
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishList4TestEtcd(PublishList4Test):
+    """
+    publish list json: empty list
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"
+
+
+class PublishList5TestEtcd(PublishList5Test):
+    """
+    publish list json: several repos list
+    """
+    databaseType = "etcd"
+    databaseUrl = "127.0.0.1:2379"

system/t13_etcd/start-etcd.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+if [ -e /tmp/etcd.pid ]; then
+    echo etcd already running, killing..
+    etcdpid=`cat /tmp/etcd.pid`
+    kill $etcdpid
+    sleep 2
+fi
+
+finish()
+{
+    if [ -n "$etcdpid" ]; then
+        echo terminating etcd
+        kill $etcdpid
+    fi
+}
+trap finish INT
+
+/srv/etcd/etcd --max-request-bytes '1073741824' --data-dir /tmp/etcd-data &
+echo $! > /tmp/etcd.pid
+etcdpid=`cat /tmp/etcd.pid`
+wait $etcdpid
+echo etcd terminated

system/t13_etcd/test_release

@@ -0,0 +1 @@
+../t04_mirror/test_release

system/t13_etcd/test_release2

@@ -0,0 +1 @@
+../t04_mirror/test_release2