Merge pull request #1436 from benallard/dput_api

Add API endpoint suitable for dput.

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
       - name: "Install Test Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime dput-ng
 
       - name: "Checkout Repository"
         uses: actions/checkout@v4

api/files.go

@@ -185,6 +185,69 @@ func apiFilesUpload(c *gin.Context) {
 	c.JSON(200, stored)
 }
 
+// @Summary Upload One File
+// @Description **Upload one file to a directory**
+// @Description
+// @Description - file is uploaded
+// @Description - existing uploaded are overwritten
+// @Description
+// @Description **Example:**
+// @Description  ```
+// @Description $ dput aptly aptly_0.9~dev+217+ge5d646c_i386.changes
+// @Description  ```
+// @Tags Files
+// @Param dir path string true "Directory to upload files to. Created if does not exist"
+// @Param file path string true "File to upload"
+// @Produce json
+// @Success 200 {array} string "Name of uploaded file"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Not Found"
+// @Failure 500 {object} Error "Internal Server Error"
+// @Router /api/files/{dir}/{file} [put]
+func apiFilesUploadOne(c *gin.Context) {
+	if !verifyDir(c) {
+		return
+	}
+
+	fileName := c.Params.ByName("file")
+	if !verifyPath(fileName) {
+		AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
+		return
+	}
+
+	path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
+	err := os.MkdirAll(path, 0777)
+
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+	stored := []string{}
+
+	destPath := filepath.Join(path, fileName)
+	dst, err := os.Create(destPath)
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+	defer dst.Close()
+
+	if _, err = io.Copy(dst, c.Request.Body); err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
+
+	if err = syncFile(dst); err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", fileName, err))
+		return
+	}
+
+	stored = append(stored, filepath.Join(c.Params.ByName("dir"), fileName))
+
+	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
+	c.JSON(200, stored)
+}
+
 // @Summary List Files
 // @Description **Show uploaded files in upload directory**
 // @Description

api/router.go

@@ -182,6 +182,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	{
 		api.GET("/files", apiFilesListDirs)
 		api.POST("/files/:dir", apiFilesUpload)
+		api.PUT("/files/:dir/:file", apiFilesUploadOne)
 		api.GET("/files/:dir", apiFilesListFiles)
 		api.DELETE("/files/:dir", apiFilesDeleteDir)
 		api.DELETE("/files/:dir/:name", apiFilesDeleteFile)

system/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b
         binutils-arm-linux-gnueabihf bash-completion zip ruby-dev lintian npm \
         libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \
         gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \
-        faketime && \
+        faketime dput-ng && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 RUN useradd -m --shell /bin/bash --home-dir /var/lib/aptly aptly

system/t12_api/files.py

@@ -1,4 +1,10 @@
+import inspect
+import os
+import shutil
+import tempfile
+
 from api_lib import APITest
+from lib import BaseTest
 
 
 class FilesAPITestUpload(APITest):
@@ -97,3 +103,76 @@ class FilesAPITestSecurity(APITest):
         self.check_equal(self.delete("/api/files/../.").status_code, 404)
         self.check_equal(self.delete("/api/files/./..").status_code, 404)
         self.check_equal(self.delete("/api/files/dir/..").status_code, 404)
+
+
+class FilesAPITestDputUpload(APITest):
+    """
+    PUT /api/files/:dir/:file via dput, then POST /api/repos/:name/include/:dir
+
+    Uses the real dput binary to upload a .changes file and all its referenced
+    files to the aptly API, then imports them into a local repo via include.
+    Skipped if dput is not installed.
+    """
+
+    def fixture_available(self):
+        return super().fixture_available() and shutil.which("dput") is not None
+
+    def check(self):
+        d = self.random_name()
+        repo_name = self.random_name()
+
+        # Create target repo
+        self.check_equal(
+            self.post("/api/repos", json={"Name": repo_name}).status_code, 201)
+
+        changes_dir = os.path.join(
+            os.path.dirname(inspect.getsourcefile(BaseTest)), "changes")
+        changes_file = os.path.join(changes_dir, "hardlink_0.2.1_amd64.changes")
+
+        # dput strips leading/trailing slashes from 'incoming' then prepends /,
+        # producing: PUT http://{fqdn}/api/files/{d}/{filename}
+        # fqdn includes host:port so dput connects directly to the test API server.
+        dput_cf = (
+            "[aptly]\n"
+            f"fqdn     = {self.base_url}\n"
+            "method   = http\n"
+            f"incoming = api/files/{d}\n"
+            "login    = *\n"
+            "allow_unsigned_uploads = 1\n"
+            "allow_dcut = 0\n"
+        )
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            dput_cf_path = os.path.join(tmpdir, "dput.cf")
+            with open(dput_cf_path, "w") as f:
+                f.write(dput_cf)
+
+            # dput -U: allow unsigned uploads (skip local GPG check)
+            # dput reads the .changes and PUTs every file listed in Files: + the .changes itself
+            self.run_cmd(["dput", "-c", dput_cf_path, "-U", "aptly", changes_file])
+        finally:
+            shutil.rmtree(tmpdir)
+
+        # All files referenced in the .changes must now be present in the upload dir
+        self.check_exists(f"upload/{d}/hardlink_0.2.1_amd64.changes")
+        self.check_exists(f"upload/{d}/hardlink_0.2.1.dsc")
+        self.check_exists(f"upload/{d}/hardlink_0.2.1.tar.gz")
+        self.check_exists(f"upload/{d}/hardlink_0.2.1_amd64.deb")
+
+        # Import via the .changes file into the repo
+        resp = self.post_task(
+            f"/api/repos/{repo_name}/include/{d}",
+            params={"ignoreSignature": 1})
+        self.check_task(resp)
+
+        output = self.get(f"/api/tasks/{resp.json()['ID']}/output")
+        self.check_in(b"Added: hardlink_0.2.1_source added, hardlink_0.2.1_amd64 added", output.content)
+
+        # Packages must be in the repo
+        self.check_equal(
+            sorted(self.get(f"/api/repos/{repo_name}/packages").json()),
+            ["Pamd64 hardlink 0.2.1 daf8fcecbf8210ad", "Psource hardlink 0.2.1 8f72df429d7166e5"])
+
+        # include cleans up the upload dir
+        self.check_not_exists(f"upload/{d}")