Merge pull request #1352 from aptly-dev/feature/storage-api

Feature/storage api
2026-01-12 03:21:33 +00:00 · 2024-10-03 23:02:19 +02:00
parent aff7b0db50 dbc336f921
commit 880f487093
17 changed files with 128 additions and 42 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -70,7 +70,7 @@ jobs:
        with:
          directory: ${{ runner.temp }}

-      - name: Make
+      - name: "Run System Tests"
        env:
          RUN_LONG_TESTS: 'yes'
          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
@@ -82,12 +82,12 @@ jobs:
          sudo mkdir -p /srv ; sudo chown runner /srv
          COVERAGE_DIR=${{ runner.temp }} make all

-      - name: Merge code coverage
+      - name: "Merge code coverage"
        run: |
          go install github.com/wadey/gocovmerge@latest
          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt

-      - name: Upload code coverage
+      - name: "Upload code coverage"
        uses: codecov/codecov-action@v2
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
@@ -185,7 +185,7 @@ jobs:
          APTLY_USER: ${{ secrets.APTLY_USER }}
          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
        run: |
-          ./upload-artifacts.sh ci ${{ matrix.suite }}
+          .github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}

      - name: "Publish release to aptly"
        if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
@@ -193,7 +193,7 @@ jobs:
          APTLY_USER: ${{ secrets.APTLY_USER }}
          APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
        run: |
-          ./upload-artifacts.sh release ${{ matrix.suite }}
+          .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}

  ci-binary-build:
    name: "Build"
@@ -247,7 +247,8 @@ jobs:
        run: |
          make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}

-      - uses: actions/upload-artifact@v4
+      - name: "Upload Artifacts"
+        uses: actions/upload-artifact@v4
        if: startsWith(github.event.ref, 'refs/tags')
        with:
          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
--- a/.github/workflows/scripts/upload-artifacts.sh
+++ b/.github/workflows/scripts/upload-artifacts.sh
--- a/18
+++ b/18
@@ -26,7 +26,7 @@ prepare:  ## Install go module dependencies
 	go mod download
 	# install and initialize swagger
 	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	go mod verify
 	go mod tidy -v
 	go generate
@@ -70,8 +70,8 @@ docker-test: ## Run system tests
 	@rm -f aptly.test
 	go generate
 	# install and initialize swagger
-	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@latest
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# build coverage binary
 	go test -v -coverpkg="./..." -c -tags testruncli
 	@echo Running python tests ...
@@ -126,7 +126,7 @@ releasetype:  # Print release type (ci/release)
 build:  ## Build aptly
 	# install and initialize swagger
 	unset GOBIN; go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# prepare
 	go mod tidy
 	go generate
@@ -137,7 +137,7 @@ dev-server: prepare  ## Run dev-server
 	go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf /var/lib/aptly/.aptly.conf
 	sed -i /enableSwaggerEndpoint/s/false/true/ /var/lib/aptly/.aptly.conf
-	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q' -build.exclude_dir system -build.exclude_dir debian -build.exclude_dir docs -- api serve -listen 0.0.0.0:3142

 dpkg:  ## Build debian packages
 	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
@@ -145,7 +145,7 @@ dpkg:  ## Build debian packages
 	GOPATH=$$PWD/.go go generate -v
 	# install and initialize swagger
 	go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# set debian version
 	@if [ "`make -s releasetype`" = "ci" ]; then  \
 		echo CI Build, setting version... ; \
@@ -169,7 +169,7 @@ binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar)
 	@make version > VERSION
 	# install and initialize swagger
 	GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest
-	PATH=$(BINPATH)/:$(PATH) swag init
+	PATH=$(BINPATH)/:$(PATH) swag init -q
 	# build aptly
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
 	# install
@@ -194,7 +194,7 @@ docker-image:  ## Build aptly-dev docker image
 docker-build:  ## Build aptly in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd make build

-docker-aptly:  ## Build and run aptly commands in docker container
+docker-shell:  ## Run aptly and other commands in docker container
 	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/run-aptly-cmd

 docker-deb:  ## Build debian packages in docker container
@@ -222,4 +222,4 @@ clean:  ## remove local build and module cache
 	test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
 	rm -rf build/ docs/ obj-*-linux-gnu*

-.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-aptly clean releasetype dpkg dev-server docker-dev-server
+.PHONY: help man prepare version binaries docker-release docker-system-tests docker-unit-tests docker-lint docker-build docker-image build docker-shell clean releasetype dpkg dev-server docker-dev-server
--- a/api/publish.go
+++ b/api/publish.go
@@ -57,6 +57,7 @@ func parseEscapedPath(path string) string {
 // @Tags Publish
 // @Produce  json
 // @Success 200 {array} deb.PublishedRepo
+// @Failure 500 {object} Error "Internal Error"
 // @Router /api/publish [get]
 func apiPublishList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
--- a/api/router.go
+++ b/api/router.go
@@ -125,6 +125,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 			api.GET("/metrics", apiMetricsGet())
 		}
 		api.GET("/version", apiVersion)
+		api.GET("/storage", apiDiskFree)

 		isReady := &atomic.Value{}
 		isReady.Store(false)
--- a/api/storage.go
+++ b/api/storage.go
@@ -0,0 +1,43 @@
+package api
+
+import (
+	"fmt"
+	"syscall"
+
+	"github.com/gin-gonic/gin"
+)
+
+type diskFree struct {
+	// Storage size [MiB]
+	Total uint64
+	// Available Storage [MiB]
+	Free uint64
+	// Percentage Full
+	PercentFull float32
+}
+
+// @Summary Get Storage Utilization
+// @Description **Get disk free information of aptly storage**
+// @Tags Status
+// @Produce json
+// @Success 200 {object} diskFree "Storage information"
+// @Failure 400 {object} Error "Internal Error"
+// @Router /api/storage [get]
+func apiDiskFree(c *gin.Context) {
+	var df diskFree
+
+	fs := context.Config().GetRootDir()
+
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(fs, &stat)
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
+		return
+	}
+
+	df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
+	df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
+	df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
+
+	c.JSON(200, df)
+}
--- a/go.mod
+++ b/go.mod
@@ -101,8 +101,8 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/rogpeppe/go-internal v1.10.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.0-rc.0 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.15 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.15 // indirect
 	go.uber.org/multierr v1.10.0 // indirect
 	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
@@ -110,7 +110,9 @@ require (
 	golang.org/x/sync v0.8.0 // indirect
 	golang.org/x/text v0.17.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
+	google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
 	google.golang.org/grpc v1.64.1 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -127,5 +129,5 @@ require (
 	github.com/swaggo/files v1.0.1
 	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.3
-	go.etcd.io/etcd/client/v3 v3.5.0-rc.0
+	go.etcd.io/etcd/client/v3 v3.5.15
 )
--- a/go.sum
+++ b/go.sum
@@ -365,10 +365,16 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/etcd/api/v3 v3.5.0-rc.0 h1:oPIG9qBFXiGJFM1StoFd5EVllfVIhd7pPjB/mcMCzCg=
 go.etcd.io/etcd/api/v3 v3.5.0-rc.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
+go.etcd.io/etcd/api/v3 v3.5.15 h1:3KpLJir1ZEBrYuV2v+Twaa/e2MdDCEZ/70H+lzEiwsk=
+go.etcd.io/etcd/api/v3 v3.5.15/go.mod h1:N9EhGzXq58WuMllgH9ZvnEr7SI9pS0k0+DHZezGp7jM=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0 h1:7mW2vCG+tthhWwORSD/G0eesRZ6ZkOh19P7CyrT9xfA=
 go.etcd.io/etcd/client/pkg/v3 v3.5.0-rc.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
+go.etcd.io/etcd/client/pkg/v3 v3.5.15 h1:fo0HpWz/KlHGMCC+YejpiCmyWDEuIpnTDzpJLB5fWlA=
+go.etcd.io/etcd/client/pkg/v3 v3.5.15/go.mod h1:mXDI4NAOwEiszrHCb0aqfAYNCrZP4e9hRca3d1YK8EU=
 go.etcd.io/etcd/client/v3 v3.5.0-rc.0 h1:W2gUtoUho3+X5tCffYA3VfA7fH9DokrhGMscvZskuXc=
 go.etcd.io/etcd/client/v3 v3.5.0-rc.0/go.mod h1:5+cKiK2IEMa230cLFrLL3+3sKlPPKASjiLOmG8tZ8vY=
+go.etcd.io/etcd/client/v3 v3.5.15 h1:23M0eY4Fd/inNv1ZfU3AxrbbOdW79r9V9Rl62Nm6ip4=
+go.etcd.io/etcd/client/v3 v3.5.15/go.mod h1:CLSJxrYjvLtHsrPKsy7LmZEE+DK2ktfd2bN4RhBMwlU=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
 go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
@@ -529,6 +535,12 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4=
+google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
--- a/system/Dockerfile
+++ b/system/Dockerfile
@@ -8,6 +8,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg a
    apt-get clean && rm -rf /var/lib/apt/lists/*

 RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly
+RUN sed -i 's/#force_color_prompt=yes/force_color_prompt=yes/' /var/lib/aptly/.bashrc

 RUN mkdir /work
 WORKDIR /work
--- a/system/build-deb
+++ b/system/build-deb
@@ -5,6 +5,9 @@ chown -R `stat -c %u /work/src` /var/lib/aptly

 su aptly -c 'set -e; cd /work/src;
 GOPATH=$PWD/.go go generate -v
+# install and initialize swagger
+GOPATH=$PWD/.go go install github.com/swaggo/swag/cmd/swag@latest
+PATH=$PWD/.go/bin:$PATH swag init
 git checkout debian/changelog
 DEBEMAIL="CI <runner@github>" dch -v `make version` "CI build"
 dpkg-buildpackage -us -uc -b -d
--- a/system/run.py
+++ b/system/run.py
@@ -11,6 +11,7 @@ from tempfile import mkdtemp
 import traceback
 import random
 import subprocess
+import time

 from lib import BaseTest
 from s3_lib import S3Test
@@ -76,15 +77,15 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                if name in testignore:
                    continue

-                testout.clear()
-
                o = getattr(testModule, name)
-
                if not (inspect.isclass(o) and issubclass(o, BaseTest) and o is not BaseTest and
                        o is not SwiftTest and o is not S3Test and o is not AzureTest and
                        o is not APITest and o is not FileSystemEndpointTest):
                    continue

+                testout.clear()
+                start_time = time.time()
+
                newBase = o.__bases__[0]
                if lastBase is not None and lastBase is not newBase:
                    lastBase.shutdown_class()
@@ -102,7 +103,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
                    if not matches:
                        continue

-                orig_stdout.write("%s: %s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
+                orig_stdout.write("· %-13s ➔ %-48s ... " % (test, colored(o.__name__, color="yellow", attrs=["bold"])))
                orig_stdout.flush()

                t = o()
@@ -118,20 +119,41 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non

                numTests += 1

+                failed = False
+                t.captureResults = capture_results
+                t.coverage_dir = coverage_dir
+                typ = None
+                val = None
+                tb = None
                try:
-                    t.captureResults = capture_results
-                    t.coverage_dir = coverage_dir
                    t.test()
                except Exception:
-                    numFailed += 1
                    typ, val, tb = sys.exc_info()
+                    failed = True
+
+                end_time = time.time()
+                execution_time = int(end_time - start_time) + 1
+                minutes = execution_time // 60
+                seconds = execution_time % 60
+                if minutes > 0:
+                    minutes = f"{minutes}m"
+                    if seconds < 10:
+                        seconds = f"0{seconds}"
+                else:
+                    minutes = "  "
+                    if seconds < 10:
+                        seconds = f" {seconds}"
+                duration = f"{minutes}{seconds}s"
+
+                if failed:
+                    numFailed += 1
                    fails.append((test, t, typ, val, tb, testModule))
-                    orig_stdout.write(colored("\b\b\b\bFAIL\n", color="red", attrs=["bold"]))
+                    orig_stdout.write(colored("\b\b\b\bFAIL", color="red", attrs=["bold"]) + f"   {duration}\n")

                    orig_stdout.write(testout.get_contents())
                    traceback.print_exception(typ, val, tb, file=orig_stdout)
                else:
-                    orig_stdout.write(colored("\b\b\b\bOK \n", color="green", attrs=["bold"]))
+                    orig_stdout.write(colored("\b\b\b\bOK", color="green", attrs=["bold"]) + f"   {duration}\n")

                t.shutdown()

--- a/system/t04_mirror/UpdateMirror11FTPTest_gold
+++ b/system/t04_mirror/UpdateMirror11FTPTest_gold
--- a/system/t04_mirror/edit.py
+++ b/system/t04_mirror/edit.py
@@ -112,6 +112,5 @@ class EditMirror10Test(BaseTest):
    """
    edit mirror: change archive url
    """
-    requiresFTP = True
    fixtureCmds = ["aptly mirror create -ignore-signatures mirror10 http://repo.aptly.info/system-tests/ftp.ru.debian.org/debian bookworm main"]
    runCmd = "aptly mirror edit -ignore-signatures -archive-url http://repo.aptly.info/system-tests/ftp.ch.debian.org/debian mirror10"
--- a/system/t04_mirror/update.py
+++ b/system/t04_mirror/update.py
@@ -162,7 +162,7 @@ class UpdateMirror10Test(BaseTest):
    outputMatchPrepare = filterOutSignature


-class UpdateMirror11Test(BaseTest):
+class UpdateMirror11FTPTest(BaseTest):
    """
    update mirrors: update over FTP
    """
--- a/system/t06_publish/repo.py
+++ b/system/t06_publish/repo.py
@@ -671,12 +671,12 @@ class PublishRepo26Test(BaseTest):
    """
    publish repo: sign with passphrase
    """
-    skipTest = "Failing on CI"
    fixtureCmds = [
        "aptly repo create local-repo",
        "aptly repo add local-repo ${files}",
+        "gpg --import --batch --passphrase verysecret ${files}/aptly_passphrase.sec"
    ]
-    runCmd = "aptly publish repo -keyring=${files}/aptly_passphrase.pub -secret-keyring=${files}/aptly_passphrase.sec -passphrase=verysecret -distribution=maverick local-repo"
+    runCmd = "aptly publish repo -batch -keyring=${files}/aptly_passphrase.pub -passphrase=verysecret -distribution=maverick local-repo"
    gold_processor = BaseTest.expand_environ

    def outputMatchPrepare(_, s):
--- a/system/t12_api/storage.py
+++ b/system/t12_api/storage.py
@@ -0,0 +1,11 @@
+from api_lib import APITest
+
+
+class TaskAPITestSwaggerDocs(APITest):
+    """
+    GET /docs
+    """
+
+    def check(self):
+        resp = self.get("/api/storage")
+        self.check_equal(resp.status_code, 200)
--- a/system/t13_etcd/mirror_update.py
+++ b/system/t13_etcd/mirror_update.py
@@ -9,7 +9,6 @@ from t04_mirror.update import UpdateMirror1Test, \
        UpdateMirror8Test, \
        UpdateMirror9Test, \
        UpdateMirror10Test, \
-        UpdateMirror11Test, \
        UpdateMirror12Test, \
        UpdateMirror13Test, \
        UpdateMirror14Test, \
@@ -34,7 +33,6 @@ TEST_IGNORE = ["UpdateMirror1Test",
               "UpdateMirror8Test",
               "UpdateMirror9Test",
               "UpdateMirror10Test",
-               "UpdateMirror11Test",
               "UpdateMirror12Test",
               "UpdateMirror13Test",
               "UpdateMirror14Test",
@@ -130,14 +128,6 @@ class UpdateMirror10TestEtcd(UpdateMirror10Test):
    databaseUrl = "127.0.0.1:2379"


-class UpdateMirror11TestEtcd(UpdateMirror11Test):
-    """
-    update mirrors: update over FTP
-    """
-    databaseType = "etcd"
-    databaseUrl = "127.0.0.1:2379"
-
-
 class UpdateMirror12TestEtcd(UpdateMirror12Test):
    """
    update mirrors: update with udebs