feat: Add system test for etcd

Makefile

@@ -12,11 +12,7 @@ COVERAGE_DIR?=$(shell mktemp -d)
 # Uncomment to update test outputs
 # CAPTURE := "--capture"
 
-# etcd test env
-ETCD_VER=v3.5.2
-DOWNLOAD_URL=https://storage.googleapis.com/etcd
-
-all: modules test bench check system-test
+all: modules test bench check system-test system-test-etcd
 
 # Self-documenting Makefile
 # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
@@ -26,13 +22,6 @@ help:  ## Print this help
 prepare:
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
 
-etcd-prepare:
-	# etcd test prepare
-	rm -rf /tmp/etcd-download-test/test-data && mkdir -p /tmp/etcd-download-test/test-data
-	if [ ! -e /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz ]; then curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz; fi
-	tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
-	/tmp/etcd-download-test/etcd --data-dir /tmp/etcd-download-test/test-data &
-
 modules:
 	go mod download
 	go mod verify
@@ -79,9 +68,18 @@ docker-test: install
 	export APTLY_VERSION=$(VERSION); \
 	$(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
 
-test: etcd-prepare
+test:
 	go test -v ./... -gocheck.v=true -coverprofile=unit.out
 
+system-test-etcd: install system/env
+ifeq ($(RUN_LONG_TESTS), yes)
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	# TODO: maybe we can skip imgrading levledb data to etcd
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/leveldb2etcd.py --datadir ~/aptly-fixture-db
+	PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_ETCD_DATABASE="127.0.0.1:2379" APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
+endif
+
 bench:
 	go test -v ./deb -run=nothing -bench=. -benchmem
 

system/leveldb2etcd.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+
+import leveldb
+import etcd3
+import argparse
+from termcolor import cprint
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--datadir", required=True, help="leveldb data dir")
+    parser.add_argument("--etcdaddr", default="127.0.0.1", help="etcd server address")
+    parser.add_argument("--etcdport", default="2379", help="etcd server address")
+
+    args = parser.parse_args()
+
+    ldb = leveldb.LevelDB(args.datadir)
+    etcd = etcd3.client(args.etcdaddr, args.etcdport)
+
+    for key, value in ldb.RangeIter():
+        try:
+            keystr = str(bytes(key))
+            valuestr = str(bytes(value))
+            etcd.put(keystr, valuestr)
+            # cprint("key: "+keystr+", value: "+valuestr+"put success!\n", 'green')
+        except Exception as e:
+            cprint("key: " + keystr + ", value: " + valuestr + "put err: " + str(e) + "\n", 'red')
+            exit(1)

system/lib.py

@@ -133,6 +133,10 @@ class BaseTest(object):
     aptlyDir = ".aptly"
     aptlyConfigFile = ".aptly.conf"
     expectedCode = 0
+    databaseEtcd = os.environ.get("APTLY_ETCD_DATABASE")
+    if databaseEtcd is None:
+        databaseEtcd = ""
+
     configFile = {
         "rootDir": f"{os.environ['HOME']}/{aptlyDir}",
         "downloadConcurrency": 4,
@@ -151,7 +155,8 @@ class BaseTest(object):
         "enableMetricsEndpoint": True,
         "logLevel": "debug",
         "logFormat": "default",
-        "serveInAPIMode": True
+        "serveInAPIMode": True,
+        "databaseEtcd": databaseEtcd,
     }
     configOverride = {}
     environmentOverride = {}

system/requirements.txt

@@ -5,3 +5,5 @@ requests-unixsocket
 python-swiftclient
 flake8
 termcolor
+etcd3
+leveldb