(not4debian) ci: run unit tests in docker

- run separate unit-test job
- build docker
- allow make docker-unit-tests in ci

.github/workflows/ci.yml

@@ -17,6 +17,29 @@ env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
+  unit-test:
+    name: "Unit Tests (Debian 13)"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+      - name: "Docker Image"
+        run: |
+          make docker-image
+      - name: "Unit Test"
+        run: |
+          make docker-unit-test
+      - name: "Upload Code Coverage"
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: unit.out
+
   test:
     name: "Test (Ubuntu 22.04)"
     runs-on: ubuntu-22.04
@@ -63,18 +86,6 @@ jobs:
         with:
           directory: ${{ runner.temp }}
 
-      - name: "Run Unit Tests"
-        env:
-          RUN_LONG_TESTS: 'yes'
-          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
-          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
-          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        run: |
-          sudo mkdir -p /srv ; sudo chown runner /srv
-          COVERAGE_DIR=${{ runner.temp }} make test
-
       - name: "Run Benchmark"
         run: |
           COVERAGE_DIR=${{ runner.temp }} make bench
@@ -93,8 +104,9 @@ jobs:
 
       - name: "Merge Code Coverage"
         run: |
-          go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
-          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+          # go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
+          # ~/go/bin/gocovmerge coverage/*.out > coverage.txt
+          awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
 
       - name: "Upload Code Coverage"
         uses: codecov/codecov-action@v2
@@ -104,7 +116,8 @@ jobs:
 
   ci-debian-build:
     name: "Build"
-    needs: test
+    needs:
+      - test
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -226,7 +239,9 @@ jobs:
 
   ci-binary-build:
     name: "Build"
-    needs: test
+    needs:
+      - unit-test
+      - test
     runs-on: ubuntu-latest
     strategy:
       matrix:

AUTHORS

@@ -69,3 +69,10 @@ List of contributors, in chronological order:
 * Leigh London (https://github.com/leighlondon)
 * Gordian Schoenherr (https://github.com/schoenherrg)
 * Silke Hofstra (https://github.com/silkeh)
+* Itay Porezky (https://github.com/itayporezky)
+* JupiterRider (https://github.com/JupiterRider)
+* Tobias Assarsson (https://github.com/daedaluz)
+* Yaksh Bariya (https://github.com/thunder-coding)
+* Brian Witt (https://github.com/bwitt)
+* Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu
 ### List of Repositories
 
 * [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
-* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
 

Makefile

@@ -7,9 +7,23 @@ COVERAGE_DIR?=$(shell mktemp -d)
 GOOS=$(shell go env GOHOSTOS)
 GOARCH=$(shell go env GOHOSTARCH)
 
+export PODMAN_USERNS = keep-id
+DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
+
+# Setting TZ for certificates
+export TZ=UTC
 # Unit Tests and some sysmte tests rely on expired certificates, turn back the time
 export TEST_FAKETIME := 2025-01-02 03:04:05
 
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
+else
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
+endif
+
 # export CAPUTRE=1 for regenrating test gold files
 ifeq ($(CAPTURE),1)
 CAPTURE_ARG := --capture
@@ -61,9 +75,9 @@ azurite-start:
 azurite-stop:
 	@kill `cat ~/.azurite.pid`
 
-swagger: swagger-install
+swagger: #swagger-install
 	# Generate swagger docs
-	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+	#@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
 
 etcd-install:
 	# Install etcd
@@ -101,13 +115,13 @@ test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify
 
 system-test: prepare swagger etcd-install  ## Run system tests
 	# build coverage binary
-	go test -v -coverpkg="./..." -c -tags testruncli
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
 	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
 	# Run system tests
-	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST)
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
 
 bench:
 	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -117,6 +131,7 @@ serve: prepare swagger-install  ## Run development server (auto recompiling)
 	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf ~/.aptly.conf
 	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
 	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
 dpkg: prepare swagger  ## Build debian packages
@@ -171,16 +186,16 @@ docker-image-no-cache:  ## Build aptly-dev docker image (no cache)
 	@docker build --no-cache -f system/Dockerfile . -t aptly-dev
 
 docker-build:  ## Build aptly in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
 
 docker-shell:  ## Run aptly and other commands in docker container
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+	@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
 
 docker-deb:  ## Build debian packages in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
 
 docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
@@ -189,27 +204,27 @@ docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to spec
 		azurite-stop
 
 docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
 		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
 		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
-		system-test TEST=$(TEST) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
 		azurite-stop
 
 docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+	@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
 
 docker-lint:  ## Run golangci-lint in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
 
 docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
 
 docker-man:  ## Create man page in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
@@ -226,4 +241,4 @@ clean:  ## remove local build and module cache
 	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
 	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

api/api.go

@@ -70,7 +70,7 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) {
 			return
 		}
 
-                status := aptlyStatus{Status: "Aptly is ready"}
+		status := aptlyStatus{Status: "Aptly is ready"}
 		c.JSON(200, status)
 	}
 }
@@ -178,7 +178,7 @@ func truthy(value interface{}) bool {
 	if value == nil {
 		return false
 	}
-        switch v := value.(type) {
+	switch v := value.(type) {
 	case string:
 		switch strings.ToLower(v) {
 		case "n", "no", "f", "false", "0", "off":

api/files.go

@@ -13,6 +13,10 @@ import (
 	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 func verifyPath(path string) bool {
 	path = filepath.Clean(path)
 	for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -114,34 +118,69 @@ func apiFilesUpload(c *gin.Context) {
 	}
 
 	stored := []string{}
+	openFiles := []*os.File{}
 
+	// Write all files first
 	for _, files := range c.Request.MultipartForm.File {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer func() { _ = src.Close() }()
 
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
+				_ = src.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer func() { _ = dst.Close() }()
 
 			_, err = io.Copy(dst, src)
+			_ = src.Close()
 			if err != nil {
+				_ = dst.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
 
+			// Keep file open for batch sync
+			openFiles = append(openFiles, dst)
 			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
 		}
 	}
 
+	// Sync all files at once to catch ENOSPC errors
+	for i, dst := range openFiles {
+		err := syncFile(dst)
+		if err != nil {
+			// Close all files
+			for _, f := range openFiles {
+				_ = f.Close()
+			}
+			AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
+			return
+		}
+	}
+
+	// Close all files
+	for _, dst := range openFiles {
+		_ = dst.Close()
+	}
+
 	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
 }

api/files_test.go

@@ -0,0 +1,476 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+type FilesUploadDiskFullSuite struct {
+	aptlyContext *ctx.AptlyContext
+	flags        *flag.FlagSet
+	configFile   *os.File
+	router       http.Handler
+}
+
+var _ = Suite(&FilesUploadDiskFullSuite{})
+
+func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
+	aptly.Version = "testVersion"
+
+	file, err := os.CreateTemp("", "aptly")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+		"rootDir":       c.MkDir(),
+	})
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+	_ = file.Close()
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	aptlyContext, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.aptlyContext = aptlyContext
+	s.router = Router(aptlyContext)
+	context = aptlyContext
+}
+
+func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.aptlyContext != nil {
+		s.aptlyContext.Shutdown()
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
+	testContent := []byte("test file content for upload")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "testfile.txt")
+	c.Assert(err, IsNil)
+
+	_, err = part.Write(testContent)
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
+	testContent := bytes.Repeat([]byte("A"), 10000)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "largefile.bin")
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(part, bytes.NewReader(testContent))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir2", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(len(content), Equals, len(testContent))
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
+	testFiles := map[string][]byte{
+		"file1.txt": []byte("content of file 1"),
+		"file2.txt": bytes.Repeat([]byte("B"), 5000),
+		"file3.deb": []byte("debian package content"),
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	for filename, content := range testFiles {
+		part, err := writer.CreateFormFile("file", filename)
+		c.Assert(err, IsNil)
+		_, err = part.Write(content)
+		c.Assert(err, IsNil)
+	}
+
+	err := writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/multitest", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
+	for filename, expectedContent := range testFiles {
+		uploadedFile := filepath.Join(uploadDir, filename)
+		content, err := os.ReadFile(uploadedFile)
+		c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
+		c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
+	oldSyncFile := syncFile
+	syncFile = func(f *os.File) error {
+		if filepath.Base(f.Name()) == "syncfail.txt" {
+			return syscall.ENOSPC
+		}
+		return nil
+	}
+	defer func() { syncFile = oldSyncFile }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part1, err := writer.CreateFormFile("file", "ok.txt")
+	c.Assert(err, IsNil)
+	_, err = part1.Write([]byte("ok"))
+	c.Assert(err, IsNil)
+
+	part2, err := writer.CreateFormFile("file", "syncfail.txt")
+	c.Assert(err, IsNil)
+	_, err = part2.Write([]byte("will fail on sync"))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+	c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
+	c.Check(verifyPath("a/b/c"), Equals, true)
+	c.Check(verifyPath("../x"), Equals, false)
+	c.Check(verifyPath("./x"), Equals, true)
+	c.Check(verifyPath(".."), Equals, false)
+	c.Check(verifyPath("."), Equals, false)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
+	uploadRoot := s.aptlyContext.UploadPath()
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "d1"), Equals, true)
+	c.Check(strings.Contains(body, "d2"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
+	req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 404)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files/dir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "a.txt"), Equals, true)
+	c.Check(strings.Contains(body, "b.txt"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(base)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(filepath.Join(base, "a.txt"))
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
+	req, err := http.NewRequest("DELETE", "/api/files/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.Chmod(base, 0), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	req, err := http.NewRequest("GET", "/api/files/noperms", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	subdir := filepath.Join(base, "subdir")
+	c.Assert(os.MkdirAll(subdir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
+	req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/..", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
+	// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
+	form := &multipart.Form{
+		File: map[string][]*multipart.FileHeader{
+			"file": {{Filename: "broken.bin"}},
+		},
+	}
+
+	req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
+	c.Assert(err, IsNil)
+	req.MultipartForm = form
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.Chmod(base, 0555), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/readonly", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
+	parent := s.aptlyContext.UploadPath()
+	base := filepath.Join(parent, "cantremove")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	c.Assert(os.Chmod(parent, 0555), IsNil)
+	defer func() { _ = os.Chmod(parent, 0777) }()
+
+	req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}

api/gpg.go

@@ -28,7 +28,7 @@ type gpgAddKeyParams struct {
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
-// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description Add GPG public keys for verifying remote repositories for mirroring.
 // @Description
 // @Description Keys can be added in two ways:
 // @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)

api/mirror.go

@@ -175,9 +175,9 @@ func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	mirrorCollection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
@@ -187,21 +187,34 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+			// Fresh checks with current collections
+			snapshots := taskSnapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
-		err = mirrorCollection.Drop(repo)
+		err = taskMirrorCollection.Drop(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
@@ -232,6 +245,7 @@ func apiMirrorsShow(c *gin.Context) {
 	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
 	}
 
 	c.JSON(200, repo)
@@ -333,26 +347,8 @@ func apiMirrorsPackages(c *gin.Context) {
 type mirrorUpdateParams struct {
 	// Change mirror name to `Name`
 	Name string `                 json:"Name"                   example:"mirror1"`
-	// Url of the archive to mirror
-	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
-	// Package query that is applied to mirror packages
-	Filter string `               json:"Filter"                 example:"xserver-xorg"`
-	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
-	Architectures []string `      json:"Architectures"          example:"amd64"`
-	// Components to mirror, if not specified aptly would fetch all components
-	Components []string `         json:"Components"             example:"main"`
-	// Gpg keyring(s) for verifing Release file
+	// Gpg keyring(s) for verifying Release file
 	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
-	// Set "true" to include dependencies of matching packages when filtering
-	FilterWithDeps bool `         json:"FilterWithDeps"`
-	// Set "true" to mirror source packages
-	DownloadSources bool `        json:"DownloadSources"`
-	// Set "true" to mirror udeb files
-	DownloadUdebs bool `          json:"DownloadUdebs"`
-	// Set "true" to skip checking if the given components are in the Release file
-	SkipComponentCheck bool `     json:"SkipComponentCheck"`
-	// Set "true" to skip checking if the given architectures are in the Release file
-	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
 	// Set "true" to ignore checksum errors
 	IgnoreChecksums bool `        json:"IgnoreChecksums"`
 	// Set "true" to skip the verification of Release file signatures
@@ -387,21 +383,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
-	remote, err = collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	remote, err = collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	b.Name = remote.Name
-	b.DownloadUdebs = remote.DownloadUdebs
-	b.DownloadSources = remote.DownloadSources
-	b.SkipComponentCheck = remote.SkipComponentCheck
-	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
-	b.FilterWithDeps = remote.FilterWithDeps
-	b.Filter = remote.Filter
-	b.Architectures = remote.Architectures
-	b.Components = remote.Components
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
 	log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -410,6 +399,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Pre-task validation of new name if provided
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
@@ -418,27 +408,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
-	if b.DownloadUdebs != remote.DownloadUdebs {
-		if remote.IsFlat() && b.DownloadUdebs {
-			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
-			return
-		}
-	}
-
-	if b.ArchiveURL != "" {
-		remote.SetArchiveRoot(b.ArchiveURL)
-	}
-
-	remote.Name = b.Name
-	remote.DownloadUdebs = b.DownloadUdebs
-	remote.DownloadSources = b.DownloadSources
-	remote.SkipComponentCheck = b.SkipComponentCheck
-	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
-	remote.FilterWithDeps = b.FilterWithDeps
-	remote.Filter = b.Filter
-	remote.Architectures = b.Architectures
-	remote.Components = b.Components
-
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -447,9 +416,26 @@ func apiMirrorsUpdate(c *gin.Context) {
 
 	resources := []string{string(remote.Key())}
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.RemoteRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		remote, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Fresh rename check inside lock (if renaming)
+		if b.Name != remote.Name {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: mirror %s already exists", b.Name)
+			}
+		}
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
+		err = remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -461,7 +447,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, taskCollectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -480,8 +466,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(),
+			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -491,12 +477,12 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				_ = collection.Update(remote)
+				_ = taskCollection.Update(remote)
 			}
 		}()
 
 		remote.MarkAsUpdating()
-		err = collection.Update(remote)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -600,7 +586,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 						}
 
 						// and import it back to the pool
-						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, taskCollectionFactory.ChecksumCollection(nil))
 						if err != nil {
 							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
 							pushError(err)
@@ -653,8 +639,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		log.Info().Msgf("%s: Finalizing download...", b.Name)
-		_ = remote.FinalizeDownload(collectionFactory, out)
-		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		_ = remote.FinalizeDownload(taskCollectionFactory, out)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

api/publish.go

@@ -16,8 +16,8 @@ import (
 type signingParams struct {
 	// Don't sign published repository
 	Skip bool `                json:"Skip"           example:"false"`
-	// GPG key ID to use when signing the release, if not specified default key is used
-	GpgKey string `            json:"GpgKey"         example:"A0546A43624A8331"`
+	// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used
+	GpgKey string `            json:"GpgKey"         example:"KEY_ID_a, KEY_ID_b"`
 	// GPG keyring to use (instead of default)
 	Keyring string `           json:"Keyring"        example:"trustedkeys.gpg"`
 	// GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -41,7 +41,21 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(options.GpgKey)
+
+	var multiGpgKeys []string
+	// REST params have priority over config
+	if options.GpgKey != "" {
+		for _, p := range strings.Split(options.GpgKey, ",") {
+			if t := strings.TrimSpace(p); t != "" {
+				multiGpgKeys = append(multiGpgKeys, t)
+			}
+		}
+	} else if len(context.Config().GpgKeys) > 0 {
+		multiGpgKeys = context.Config().GpgKeys
+	}
+	for _, gpgKey := range multiGpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
 
@@ -110,7 +124,7 @@ func apiPublishList(c *gin.Context) {
 // @Description See also: `aptly publish show`
 // @Tags Publish
 // @Produce json
-// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs"
+// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs"
 // @Param distribution path string true "distribution name"
 // @Success 200 {object} deb.PublishedRepo
 // @Failure 404 {object} Error "Published repository not found"
@@ -146,10 +160,6 @@ type publishedRepoCreateParams struct {
 	Sources []sourceParams `binding:"required"    json:"Sources"`
 	// Distribution name, if missing Aptly would try to guess from sources
 	Distribution string `                         json:"Distribution"          example:"bookworm"`
-	// Value of Label: field in published repository stanza
-	Label string `                                json:"Label"                 example:""`
-	// Value of Origin: field in published repository stanza
-	Origin string `                               json:"Origin"                example:""`
 	// when publishing, overwrite files in pool/ directory without notice
 	ForceOverwrite bool `                         json:"ForceOverwrite"        example:"false"`
 	// Override list of published architectures
@@ -182,7 +192,7 @@ type publishedRepoCreateParams struct {
 // @Description **Example:**
 // @Description ```
 // @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos
-// @Description {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
+// @Description {"Architectures":["i386"],"Distribution":"wheezy","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
 // @Description ```
 // @Description
 // @Description See also: `aptly publish create`
@@ -249,7 +259,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 				return
 			}
 
-			resources = append(resources, string(snapshot.ResourceKey()))
+			resources = append(resources, string(snapshot.Key()))
 			sources = append(sources, snapshot)
 		}
 	} else if b.SourceKind == deb.SourceLocalRepo {
@@ -280,11 +290,24 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		multiDist = *b.MultiDist
 	}
 
-	collection := collectionFactory.PublishedRepoCollection()
+	// Non-MultiDist publishes share a single pool/ directory under the
+	// prefix.  Lock at the prefix level so that concurrent publish/drop
+	// operations on sibling distributions cannot race during cleanup.
+	if !multiDist {
+		storagePrefix := prefix
+		if storage != "" {
+			storagePrefix = storage + ":" + prefix
+		}
+
+		resources = append(resources, deb.PrefixPoolLockKey(storagePrefix))
+	}
 
 	taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"",
 		b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`))
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
 		taskDetail := task.PublishDetail{
 			Detail: detail,
 		}
@@ -296,10 +319,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		for _, source := range sources {
 			switch s := source.(type) {
 			case *deb.Snapshot:
-				snapshotCollection := collectionFactory.SnapshotCollection()
+				snapshotCollection := taskCollectionFactory.SnapshotCollection()
 				err = snapshotCollection.LoadComplete(s)
 			case *deb.LocalRepo:
-				localCollection := collectionFactory.LocalRepoCollection()
+				localCollection := taskCollectionFactory.LocalRepoCollection()
 				err = localCollection.LoadComplete(s)
 			default:
 				err = fmt.Errorf("unexpected type for source: %T", source)
@@ -309,23 +332,17 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			}
 		}
 
-		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist)
+		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, taskCollectionFactory, multiDist)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		resources = append(resources, string(published.Key()))
-
-		if b.Origin != "" {
-			published.Origin = b.Origin
-		}
 		if b.NotAutomatic != "" {
 			published.NotAutomatic = b.NotAutomatic
 		}
 		if b.ButAutomaticUpgrades != "" {
 			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
 		}
-		published.Label = b.Label
 
 		published.SkipContents = context.Config().SkipContentsPublishing
 		if b.SkipContents != nil {
@@ -341,18 +358,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			published.AcquireByHash = *b.AcquireByHash
 		}
 
-		duplicate := collection.CheckDuplicate(published)
+		duplicate := taskCollection.CheckDuplicate(published)
 		if duplicate != nil {
-			_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			_ = taskCollectionFactory.PublishedRepoCollection().LoadComplete(duplicate, taskCollectionFactory)
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		err = collection.Add(published)
+		err = taskCollection.Add(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -393,7 +410,6 @@ type publishedRepoUpdateSwitchParams struct {
 // @Description
 // @Description See also: `aptly publish update` / `aptly publish switch`
 // @Tags Publish
-// @Produce json
 // @Param prefix path string true "publishing prefix"
 // @Param distribution path string true "distribution name"
 // @Param _async query bool false "Run in background and return task object"
@@ -425,6 +441,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
@@ -432,48 +449,76 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
+	resources := []string{string(published.Key())}
+
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
-	} else if published.SourceKind == deb.SourceSnapshot {
-		for _, snapshotInfo := range b.Snapshots {
-			_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
 			if err2 != nil {
 				AbortWithJSONError(c, http.StatusNotFound, err2)
 				return
 			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, snapshotInfo := range b.Snapshots {
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
 		}
 	} else {
 		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
 	}
 
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
-	resources := []string{string(published.Key())}
+	// Field mutations and fresh DB load are deferred to inside the task so
+	// they always operate on a consistent state after the lock is held.
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(published, collectionFactory)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
+		}
+
 		revision := published.ObtainRevision()
 		sources := revision.Sources
 
@@ -485,17 +530,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			}
 		}
 
-		result, err := published.Update(collectionFactory, out)
+		result, err := published.Update(taskCollectionFactory, out)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -503,10 +548,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
@@ -551,10 +605,19 @@ func apiPublishDrop(c *gin.Context) {
 	}
 
 	resources := []string{string(published.Key())}
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that a drop cannot race
+	// with a concurrent update or drop of a sibling distribution during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
 	taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.Remove(context, storage, prefix, distribution,
-			collectionFactory, out, force, skipCleanup)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		err := taskCollection.Remove(context, storage, prefix, distribution,
+			taskCollectionFactory, out, force, skipCleanup)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
 		}
@@ -590,43 +653,52 @@ func apiPublishAddSource(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly (no LoadComplete) to verify existence and obtain the
+	// resource key and task name.  The actual mutation is performed inside
+	// the task on a freshly loaded copy to prevent lost-update races.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	component := b.Component
-	name := b.Name
-
-	_, exists := sources[component]
-	if exists {
-		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component))
-		return
-	}
-
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		component := b.Component
+		name := b.Name
+
+		_, exists := sources[component]
+		if exists {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("unable to create: Component '%s' already exists", component)
+		}
+
+		sources[component] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -708,39 +780,48 @@ func apiPublishSetSources(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := make(map[string]string, len(b))
-	revision.Sources = sources
-
-	for _, source := range b {
-		component := source.Component
-		name := source.Name
-		sources[component] = name
-	}
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := make(map[string]string, len(b))
+		revision.Sources = sources
+
+		for _, source := range b {
+			component := source.Component
+			name := source.Name
+			sources[component] = name
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -773,24 +854,33 @@ func apiPublishDropChanges(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and DropRevision happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	published.DropRevision()
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		published.DropRevision()
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -826,51 +916,58 @@ func apiPublishUpdateSource(c *gin.Context) {
 	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
-	component := slashEscape(c.Params.ByName("component"))
+	urlComponent := slashEscape(c.Params.ByName("component"))
+
+	// Default component to the URL path segment; the body may rename it.
+	b.Component = urlComponent
+	if c.Bind(&b) != nil {
+		return
+	}
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component))
-		return
-	}
-
-	b.Component = component
-	b.Name = revision.Sources[component]
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	if b.Component != component {
-		delete(sources, component)
-	}
-
-	component = b.Component
-	name := b.Name
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[urlComponent]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: Component '%s' does not exist", urlComponent)
+		}
+
+		if b.Component != urlComponent {
+			delete(sources, urlComponent)
+		}
+
+		newComponent := b.Component
+		name := b.Name
+		sources[newComponent] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -907,33 +1004,41 @@ func apiPublishRemoveSource(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component))
-		return
-	}
-
-	delete(sources, component)
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[component]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: Component '%s' does not exist", component)
+		}
+
+		delete(sources, component)
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -997,48 +1102,92 @@ func apiPublishUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and field mutations happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
-	}
-
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
 	resources := []string{string(published.Key())}
+
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
+
+	// Lock source repos / snapshots the same way apiPublishUpdateSwitch does,
+	// because published.Update() reads from them and concurrent modification
+	// would produce an inconsistent view.
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, uuid := range published.Sources {
+			snapshot, err2 := snapshotCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
+		}
+	}
+
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		result, err := published.Update(collectionFactory, out)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
+		}
+
+		result, err := published.Update(taskCollectionFactory, out)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -1046,10 +1195,19 @@ func apiPublishUpdate(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil

api/published_file_missing_test.go

@@ -0,0 +1,737 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+// PublishedFileMissingSuite reproduces the exact bug where:
+// - Package import succeeds
+// - Metadata is updated (Packages.gz shows the package)
+// - Publish reports success
+// - BUT the .deb file is missing from the published pool directory
+// - Result: apt-get returns 404 when trying to download the package
+type PublishedFileMissingSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+	tempDir    string
+	poolPath   string
+	publicPath string
+}
+
+var _ = Suite(&PublishedFileMissingSuite{})
+
+func (s *PublishedFileMissingSuite) SetUpSuite(c *C) {
+	aptly.Version = "publishedFileMissingTest"
+
+	tempDir, err := os.MkdirTemp("", "aptly-published-missing-test")
+	c.Assert(err, IsNil)
+	s.tempDir = tempDir
+	s.poolPath = filepath.Join(tempDir, "pool")
+	s.publicPath = filepath.Join(tempDir, "public")
+
+	file, err := os.CreateTemp("", "aptly-published-missing-config")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	config := gin.H{
+		"rootDir":                    tempDir,
+		"downloadDir":                filepath.Join(tempDir, "download"),
+		"architectures":              []string{"amd64"},
+		"dependencyFollowSuggests":   false,
+		"dependencyFollowRecommends": false,
+		"gpgDisableSign":             true,
+		"gpgDisableVerify":           true,
+		"gpgProvider":                "internal",
+		"skipLegacyPool":             true,
+		"enableMetricsEndpoint":      false,
+	}
+
+	jsonString, err := json.Marshal(config)
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+
+	flags := flag.NewFlagSet("publishedFileMissingTestFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", true, "disable database locking for test")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "config file")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *PublishedFileMissingSuite) TearDownSuite(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.context != nil {
+		s.context.Shutdown()
+	}
+	if s.tempDir != "" {
+		_ = os.RemoveAll(s.tempDir)
+	}
+}
+
+func (s *PublishedFileMissingSuite) SetUpTest(c *C) {
+	collectionFactory := s.context.NewCollectionFactory()
+
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+	_ = localRepoCollection.ForEach(func(repo *deb.LocalRepo) error {
+		_ = localRepoCollection.Drop(repo)
+		return nil
+	})
+
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+	_ = publishedCollection.ForEach(func(published *deb.PublishedRepo) error {
+		_ = publishedCollection.Remove(s.context, published.Storage, published.Prefix,
+			published.Distribution, collectionFactory, nil, true, true)
+		return nil
+	})
+}
+
+func (s *PublishedFileMissingSuite) TearDownTest(c *C) {
+	s.SetUpTest(c)
+}
+
+func (s *PublishedFileMissingSuite) httpRequest(c *C, method string, url string, body []byte) *httptest.ResponseRecorder {
+	w := httptest.NewRecorder()
+	var req *http.Request
+	var err error
+
+	if body != nil {
+		req, err = http.NewRequest(method, url, bytes.NewReader(body))
+	} else {
+		req, err = http.NewRequest(method, url, nil)
+	}
+	c.Assert(err, IsNil)
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w
+}
+
+func (s *PublishedFileMissingSuite) createDebPackage(c *C, uploadID, packageName, version string) {
+	uploadPath := s.context.UploadPath()
+	uploadDir := filepath.Join(uploadPath, uploadID)
+	err := os.MkdirAll(uploadDir, 0755)
+	c.Assert(err, IsNil)
+
+	tempDir, err := os.MkdirTemp("", "deb-build")
+	c.Assert(err, IsNil)
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	debianDir := filepath.Join(tempDir, "DEBIAN")
+	err = os.MkdirAll(debianDir, 0755)
+	c.Assert(err, IsNil)
+
+	controlContent := fmt.Sprintf(`Package: %s
+Version: %s
+Section: libs
+Priority: optional
+Architecture: amd64
+Maintainer: Test <test@example.com>
+Description: Test package
+ Test package for published file missing bug.
+`, packageName, version)
+
+	err = os.WriteFile(filepath.Join(debianDir, "control"), []byte(controlContent), 0644)
+	c.Assert(err, IsNil)
+
+	usrDir := filepath.Join(tempDir, "usr", "lib")
+	err = os.MkdirAll(usrDir, 0755)
+	c.Assert(err, IsNil)
+	err = os.WriteFile(filepath.Join(usrDir, "lib.so"), []byte("library"), 0644)
+	c.Assert(err, IsNil)
+
+	debFile := filepath.Join(uploadDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+	cmd := exec.Command("dpkg-deb", "--build", tempDir, debFile)
+	err = cmd.Run()
+	c.Assert(err, IsNil)
+}
+
+// TestPublishedFileGoMissing reproduces the exact production bug
+func (s *PublishedFileMissingSuite) TestPublishedFileGoMissing(c *C) {
+	c.Log("=== Reproducing: Package in metadata but 404 on download ===")
+
+	// Create and publish a repository
+	repoName := "test-repo"
+	distribution := "bullseye"
+
+	createBody, _ := json.Marshal(gin.H{
+		"Name":                repoName,
+		"DefaultDistribution": distribution,
+		"DefaultComponent":    "main",
+	})
+	resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to create repo: %s", resp.Body.String()))
+
+	publishBody, _ := json.Marshal(gin.H{
+		"SourceKind":    "local",
+		"Distribution":  distribution,
+		"Architectures": []string{"amd64"},
+		"Sources": []gin.H{
+			{"Component": "main", "Name": repoName},
+		},
+		"Signing": gin.H{"Skip": true},
+	})
+	resp = s.httpRequest(c, "POST", "/api/publish/hrt", publishBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to publish: %s", resp.Body.String()))
+
+	// Create package
+	packageName := "hrt-libblobbyclient1"
+	version := "20250926.152427+hrtdeb11"
+	uploadID := "test-upload-1"
+
+	s.createDebPackage(c, uploadID, packageName, version)
+
+	// Add package
+	resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package: %s", resp.Body.String()))
+
+	// Update publish
+	updateBody, _ := json.Marshal(gin.H{
+		"Signing":        gin.H{"Skip": true},
+		"ForceOverwrite": true,
+	})
+	resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/hrt/%s", distribution), updateBody)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String()))
+
+	// Now check if the file is actually accessible in the published location
+	publishedStorage, err := s.context.GetPublishedStorage("")
+	c.Assert(err, IsNil)
+	publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+	// Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb
+	expectedPath := filepath.Join(publicPath, "hrt", "pool", "main", "h", packageName,
+		fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+	c.Logf("Checking for published file at: %s", expectedPath)
+
+	fileInfo, err := os.Stat(expectedPath)
+	fileExists := err == nil
+
+	c.Logf("File exists: %v", fileExists)
+	if fileExists {
+		c.Logf("File size: %d bytes", fileInfo.Size())
+	}
+
+	// Check metadata
+	resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+	var packages []string
+	err = json.Unmarshal(resp.Body.Bytes(), &packages)
+	c.Assert(err, IsNil)
+	c.Logf("Packages in metadata: %d", len(packages))
+
+	// THE BUG: Metadata says package exists, but file is missing from published location
+	if len(packages) > 0 && !fileExists {
+		c.Logf("★★★ BUG REPRODUCED! ★★★")
+		c.Logf("Metadata shows %d package(s) but file is missing at: %s", len(packages), expectedPath)
+		c.Logf("This is exactly what causes: 404 Not Found [IP: 10.20.72.62 3142]")
+
+		c.Fatal("BUG CONFIRMED: Package in metadata but missing from published directory!")
+	}
+
+	c.Assert(fileExists, Equals, true, Commentf(
+		"Published file should exist at %s when package is in metadata", expectedPath))
+}
+
+// TestConcurrentPublishRace tries to trigger the race with concurrent publishes
+func (s *PublishedFileMissingSuite) TestConcurrentPublishRace(c *C) {
+	c.Log("=== Testing concurrent publish race condition ===")
+
+	const numIterations = 4
+
+	for iteration := 0; iteration < numIterations; iteration++ {
+		c.Logf("--- Iteration %d/%d ---", iteration+1, numIterations)
+
+		// Create repo
+		repoName := fmt.Sprintf("race-repo-%d", iteration)
+		distribution := fmt.Sprintf("dist-%d", iteration)
+
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoName,
+			"DefaultDistribution": distribution,
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		publishBody, _ := json.Marshal(gin.H{
+			"SourceKind":    "local",
+			"Distribution":  distribution,
+			"Architectures": []string{"amd64"},
+			"Sources": []gin.H{
+				{"Component": "main", "Name": repoName},
+			},
+			"Signing": gin.H{"Skip": true},
+		})
+		resp = s.httpRequest(c, "POST", "/api/publish/concurrent", publishBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		// Create multiple packages
+		var wg sync.WaitGroup
+		numPackages := 5
+
+		for i := 0; i < numPackages; i++ {
+			wg.Add(1)
+			go func(idx int) {
+				defer wg.Done()
+
+				packageName := fmt.Sprintf("pkg-%d-%d", iteration, idx)
+				version := "1.0.0"
+				uploadID := fmt.Sprintf("upload-%d-%d", iteration, idx)
+
+				s.createDebPackage(c, uploadID, packageName, version)
+
+				// Add package
+				resp := s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+				c.Logf("Package %d add: %d", idx, resp.Code)
+
+				// Small delay
+				time.Sleep(time.Duration(5+idx*2) * time.Millisecond)
+
+				// Publish
+				updateBody, _ := json.Marshal(gin.H{
+					"Signing":        gin.H{"Skip": true},
+					"ForceOverwrite": true,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/concurrent/%s", distribution), updateBody)
+				c.Logf("Publish %d: %d", idx, resp.Code)
+			}(i)
+		}
+
+		wg.Wait()
+		time.Sleep(100 * time.Millisecond)
+
+		// Check all packages
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+		var packages []string
+		err := json.Unmarshal(resp.Body.Bytes(), &packages)
+		c.Assert(err, IsNil)
+
+		// Check published files
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("pkg-%d-%d", iteration, i)
+			version := "1.0.0"
+
+			// Calculate pool path
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, "concurrent", "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, expectedPath)
+			}
+		}
+
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG DETECTED in iteration %d/%d! ★★★", iteration+1, numIterations)
+			c.Logf("Metadata shows %d packages, but %d files are MISSING:", len(packages), len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  [iter %d] File MISSING %d/%d: %s", iteration+1, i+1, len(missingFiles), f)
+			}
+
+			c.Fatalf("BUG REPRODUCED in iteration %d/%d: %d published files missing", iteration+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iteration+1, numIterations, numPackages)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced with current timing", numIterations)
+}
+
+// TestIdenticalPackageRace tests the specific case of identical SHA256 packages
+func (s *PublishedFileMissingSuite) TestIdenticalPackageRace(c *C) {
+	c.Log("=== AGGRESSIVE test: identical package (same SHA256) race ===")
+
+	const numIterations = 4
+	packageName := "shared-package"
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("Iteration %d/%d", iter+1, numIterations)
+
+		// Create two repos that will get the SAME package (unique per iteration)
+		repos := []string{fmt.Sprintf("identical-a-%d", iter), fmt.Sprintf("identical-b-%d", iter)}
+		dists := []string{fmt.Sprintf("dist-a-%d", iter), fmt.Sprintf("dist-b-%d", iter)}
+
+		for i := range repos {
+			createBody, _ := json.Marshal(gin.H{
+				"Name":                repos[i],
+				"DefaultDistribution": dists[i],
+				"DefaultComponent":    "main",
+			})
+			resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+			c.Assert(resp.Code, Equals, 201)
+
+			publishBody, _ := json.Marshal(gin.H{
+				"SourceKind":    "local",
+				"Distribution":  dists[i],
+				"Architectures": []string{"amd64"},
+				"Sources": []gin.H{
+					{"Component": "main", "Name": repos[i]},
+				},
+				"Signing": gin.H{"Skip": true},
+				"SkipBz2": true,
+			})
+			resp = s.httpRequest(c, "POST", "/api/publish/identical", publishBody)
+			c.Assert(resp.Code, Equals, 201)
+		}
+
+		// Create IDENTICAL package file with UNIQUE VERSION per iteration
+		version := fmt.Sprintf("1.0.%d", iter)
+		uploadID1 := fmt.Sprintf("identical-upload-1-%d", iter)
+		uploadID2 := fmt.Sprintf("identical-upload-2-%d", iter)
+
+		s.createDebPackage(c, uploadID1, packageName, version)
+
+		// Copy to second upload (same SHA256)
+		uploadPath := s.context.UploadPath()
+		src := filepath.Join(uploadPath, uploadID1, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+		destDir := filepath.Join(uploadPath, uploadID2)
+		err := os.MkdirAll(destDir, 0755)
+		c.Assert(err, IsNil)
+		dest := filepath.Join(destDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		srcData, readErr := os.ReadFile(src)
+		c.Assert(readErr, IsNil)
+		err = os.WriteFile(dest, srcData, 0644)
+		c.Assert(err, IsNil)
+
+		// Race: add and publish both simultaneously
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[0], uploadID1), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[0]), updateBody)
+		}()
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[1], uploadID2), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[1]), updateBody)
+		}()
+
+		wg.Wait()
+		time.Sleep(200 * time.Millisecond)
+		c.Logf("[iter %d] All operations complete", iter)
+
+		// Check the shared pool location
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		poolSubdir := string(packageName[0])
+		sharedPoolPath := filepath.Join(publicPath, "identical", "pool", "main", poolSubdir, packageName,
+			fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		fileInfo, err := os.Stat(sharedPoolPath)
+		fileExists := err == nil
+
+		if fileExists {
+			c.Logf("[iter %d] File EXISTS at %s (size: %d)", iter, sharedPoolPath, fileInfo.Size())
+		} else {
+			c.Logf("[iter %d] File MISSING at %s (error: %v)", iter, sharedPoolPath, err)
+		}
+
+		// Check metadata
+		var packagesA, packagesB []string
+		resp := s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[0]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesA)
+		c.Assert(err, IsNil)
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[1]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesB)
+		c.Assert(err, IsNil)
+
+		c.Logf("[iter %d] Packages in metadata: A=%d, B=%d", iter, len(packagesA), len(packagesB))
+
+		// THE BUG: Both repos show packages in metadata, but the shared pool file is missing
+		if (len(packagesA) > 0 || len(packagesB) > 0) && !fileExists {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d! ★★★", iter+1)
+			c.Logf("Packages in metadata A: %d, B: %d", len(packagesA), len(packagesB))
+			c.Logf("Shared pool file exists: %v", fileExists)
+			c.Logf("Pool path: %s", sharedPoolPath)
+
+			// List what files ARE in the pool directory
+			poolDir := filepath.Dir(sharedPoolPath)
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("Files in pool directory %s:", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s", entry.Name())
+				}
+			}
+
+			c.Fatalf("Metadata shows packages but shared pool file is missing (iteration %d)", iter+1)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced", numIterations)
+}
+
+// TestConcurrentSnapshotPublishToSamePrefix reproduces the EXACT production bug:
+// Multiple snapshots are published concurrently to the SAME prefix but different distributions.
+// Example from production logs:
+//   - trixie-pgdg published to "external/postgres-auto/trixie"
+//   - bullseye-pgdg published to "external/postgres-auto/bullseye"
+// Both share the same pool directory, causing cleanup race conditions.
+func (s *PublishedFileMissingSuite) TestConcurrentSnapshotPublishToSamePrefix(c *C) {
+	const numIterations = 4
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("--- Iteration %d/%d ---", iter+1, numIterations)
+
+		// Create two repos with different packages (simulating trixie-pgdg and bullseye-pgdg)
+		repoTrixie := fmt.Sprintf("trixie-pgdg-%d", iter)
+		repoBullseye := fmt.Sprintf("bullseye-pgdg-%d", iter)
+
+		// Create trixie repo
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoTrixie,
+			"DefaultDistribution": "trixie",
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie repo"))
+
+		// Create bullseye repo
+		createBody, _ = json.Marshal(gin.H{
+			"Name":                repoBullseye,
+			"DefaultDistribution": "bullseye",
+			"DefaultComponent":    "main",
+		})
+		resp = s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye repo"))
+
+		// Add packages to both repos
+		numPackages := 3
+
+		// Add packages to trixie repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("trixie-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoTrixie, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to trixie"))
+		}
+
+		// Add packages to bullseye repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("bullseye-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoBullseye, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to bullseye"))
+		}
+
+		// Create snapshots from both repos
+		snapshotTrixie := fmt.Sprintf("%s-snap", repoTrixie)
+		snapshotBullseye := fmt.Sprintf("%s-snap", repoBullseye)
+
+		createSnapshotBody, _ := json.Marshal(gin.H{"Name": snapshotTrixie})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoTrixie), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie snapshot"))
+
+		createSnapshotBody, _ = json.Marshal(gin.H{"Name": snapshotBullseye})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoBullseye), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye snapshot"))
+
+		// Publish both snapshots CONCURRENTLY to the SAME prefix
+		// This mimics production where both are published to "external/postgres-auto"
+		// Use the SAME prefix across all iterations to trigger the race more aggressively
+		sharedPrefix := "postgres-auto"
+
+		var wg sync.WaitGroup
+		var trixiePublishCode, bullseyePublishCode int
+
+		wg.Add(2)
+
+		// Publish or update trixie snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "trixie",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false, // Force cleanup to run
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE (this is what happens in production)
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/trixie", sharedPrefix), updateBody)
+			}
+			trixiePublishCode = resp.Code
+			c.Logf("[iter %d] Trixie publish/update completed: %d", iter, resp.Code)
+		}()
+
+		// Publish or update bullseye snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "bullseye",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/bullseye", sharedPrefix), updateBody)
+			}
+			bullseyePublishCode = resp.Code
+			c.Logf("[iter %d] Bullseye publish/update completed: %d", iter, resp.Code)
+		}()
+
+		wg.Wait()
+		time.Sleep(50 * time.Millisecond)
+
+		// Verify publishes succeeded (201 for create, 200 for update)
+		expectedCode := 201
+		if iter > 0 {
+			expectedCode = 200
+		}
+		c.Assert(trixiePublishCode, Equals, expectedCode, Commentf("Trixie publish/update should succeed"))
+		c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed"))
+
+		// Verify ALL package files exist in the published pool
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		expectedFiles := []string{}
+
+		// Check trixie packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("TRIXIE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// Check bullseye packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("BULLSEYE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// BUG: Files from one distribution are deleted by the other's cleanup
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d/%d! ★★★", iter+1, numIterations)
+			c.Logf("Both publishes to prefix '%s' succeeded, but %d files are MISSING:", sharedPrefix, len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  Missing file %d/%d: %s", i+1, len(missingFiles), f)
+			}
+
+			c.Logf("\nThis reproduces the exact production bug where:")
+			c.Logf("  1. Mirror updates complete successfully")
+			c.Logf("  2. Snapshots are created")
+			c.Logf("  3. Both snapshots publish to same prefix (different distributions)")
+			c.Logf("  4. Cleanup from one publish DELETES files from the other")
+			c.Logf("  5. Result: apt-get returns 404 when downloading packages")
+
+			// List what's actually in the pool
+			poolDir := filepath.Join(publicPath, sharedPrefix, "pool", "main")
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("\nActual pool directory contents (%s):", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s/", entry.Name())
+				}
+			}
+
+			c.Fatalf("BUG CONFIRMED (iteration %d/%d): %d files missing from shared pool",
+				iter+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iter+1, numIterations, len(expectedFiles))
+		}
+	}
+	c.Logf("✓ All %d iterations passed - no files missing", numIterations)
+}

api/repos.go

@@ -24,7 +24,7 @@ import (
 // @Tags Repos
 // @Produce html
 // @Success 200 {object} string "HTML"
-// @Router /api/repos [get]
+// @Router /repos [get]
 func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
 // @Param pkgPath path string true "Package Path" allowReserved=true
 // @Produce json
 // @Success 200 ""
-// @Router /api/{storage}/{pkgPath} [get]
+// @Router /repos/{storage}/{pkgPath} [get]
 func reposServeInAPIMode(c *gin.Context) {
 	pkgpath := c.Param("pkgPath")
 
@@ -60,7 +60,12 @@ func reposServeInAPIMode(c *gin.Context) {
 		storage = "filesystem:" + storage
 	}
 
-	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage(storage)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	c.FileFromFS(pkgpath, http.Dir(publicPath))
 }
 
@@ -93,7 +98,7 @@ type repoCreateParams struct {
 	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
 	// Default component when publishing from this local repo
 	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
-	// Snapshot name to create repoitory from (optional)
+	// Snapshot name to create repository from (optional)
 	FromSnapshot string `            json:"FromSnapshot"         example:""`
 }
 
@@ -122,63 +127,79 @@ func apiReposCreate(c *gin.Context) {
 		return
 	}
 
-	repo := deb.NewLocalRepo(b.Name, b.Comment)
-	repo.DefaultComponent = b.DefaultComponent
-	repo.DefaultDistribution = b.DefaultDistribution
-
+	// Handler: Pre-task validations (shallow)
 	collectionFactory := context.NewCollectionFactory()
 
+	var resources []string
 	if b.FromSnapshot != "" {
-		var snapshot *deb.Snapshot
-
-		snapshotCollection := collectionFactory.SnapshotCollection()
-
-		snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+		snapshot, err := collectionFactory.SnapshotCollection().ByName(b.FromSnapshot)
 		if err != nil {
 			AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
 			return
 		}
+		resources = append(resources, string(snapshot.Key()))
+	}
 
-		err = snapshotCollection.LoadComplete(snapshot)
-		if err != nil {
-			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
-			return
+	taskName := fmt.Sprintf("Create repository %s", b.Name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection and check/create ATOMIC inside task
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Check duplicate inside lock
+		if _, err := taskCollection.ByName(b.Name); err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+				fmt.Errorf("local repo with name %s already exists", b.Name)
 		}
 
-		repo.UpdateRefList(snapshot.RefList())
-	}
+		// Create repo
+		repo := deb.NewLocalRepo(b.Name, b.Comment)
+		repo.DefaultComponent = b.DefaultComponent
+		repo.DefaultDistribution = b.DefaultDistribution
 
-	localRepoCollection := collectionFactory.LocalRepoCollection()
+		if b.FromSnapshot != "" {
+			snapshotCollection := taskCollectionFactory.SnapshotCollection()
 
-	if _, err := localRepoCollection.ByName(b.Name); err == nil {
-		AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
-		return
-	}
+			snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, 
+					fmt.Errorf("source snapshot not found: %s", err)
+			}
 
-	err := localRepoCollection.Add(repo)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, err)
-		return
-	}
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, 
+					fmt.Errorf("unable to load source snapshot: %s", err)
+			}
 
-	c.JSON(http.StatusCreated, repo)
+			repo.UpdateRefList(snapshot.RefList())
+		}
+
+		err := taskCollection.Add(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: repo}, nil
+	})
 }
 
 type reposEditParams struct {
 	// Name of repository to modify
-	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
+	Name *string `                       json:"Name"                 example:"new-repo-name"`
 	// Change Comment of repository
 	Comment *string `                    json:"Comment"              example:"example repo"`
 	// Change Default Distribution for publishing
 	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
-	// Change Devault Component for publishing
+	// Change Default Component for publishing
 	DefaultComponent *string `        json:"DefaultComponent"     example:""`
 }
 
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Param name path string true "Repository name"
+// @Param name path string true "Repository name to modify"
 // @Consume  json
 // @Param request body reposEditParams true "Parameters"
 // @Produce json
@@ -191,42 +212,66 @@ func apiReposEdit(c *gin.Context) {
 	if c.Bind(&b) != nil {
 		return
 	}
-
+	// Load shallowly for 404 check and resource key.
+	// Mutation and duplicate check happen inside the task for atomicity.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
-	if b.Name != nil {
-		_, err := collection.ByName(*b.Name)
-		if err == nil {
-			// already exists
-			AbortWithJSONError(c, 404, err)
+	if b.Name != nil && *b.Name != name {
+		if _, err = collection.ByName(*b.Name); err == nil {
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: local repo %q already exists", *b.Name))
 			return
 		}
-		repo.Name = *b.Name
-	}
-	if b.Comment != nil {
-		repo.Comment = *b.Comment
-	}
-	if b.DefaultDistribution != nil {
-		repo.DefaultDistribution = *b.DefaultDistribution
-	}
-	if b.DefaultComponent != nil {
-		repo.DefaultComponent = *b.DefaultComponent
 	}
 
-	err = collection.Update(repo)
-	if err != nil {
-		AbortWithJSONError(c, 500, err)
-		return
-	}
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Edit repository %s", name)
 
-	c.JSON(200, repo)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		// Check and update ATOMIC (inside lock)
+		if b.Name != nil && *b.Name != name {
+			_, err := taskCollection.ByName(*b.Name)
+			if err == nil {
+				// already exists
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+					fmt.Errorf("local repo with name %q already exists", *b.Name)
+			}
+			repo.Name = *b.Name
+		}
+		if b.Comment != nil {
+			repo.Comment = *b.Comment
+		}
+		if b.DefaultDistribution != nil {
+			repo.DefaultDistribution = *b.DefaultDistribution
+		}
+		if b.DefaultComponent != nil {
+			repo.DefaultComponent = *b.DefaultComponent
+		}
+
+		err = taskCollection.Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
 }
 
 // GET /api/repos/:name
@@ -268,10 +313,10 @@ func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	name := c.Params.ByName("name")
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full checks (published/snapshots) happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	repo, err := collection.ByName(name)
 	if err != nil {
@@ -282,19 +327,32 @@ func apiReposDrop(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.ByLocalRepo(repo)
+		// Task: Create fresh collections inside task after lock acquired
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Re-read repo with fresh collection after lock
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		// Check with fresh collections
+		published := taskPublishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			snapshots := taskSnapshotCollection.ByLocalRepoSource(repo)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
 			}
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, taskCollection.Drop(repo)
 	})
 }
 
@@ -351,10 +409,13 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
@@ -363,13 +424,23 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	resources := []string{string(repo.Key())}
 
 	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		out.Printf("Loading packages...\n")
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -378,7 +449,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		for _, ref := range b.PackageRefs {
 			var p *deb.Package
 
-			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			p, err = taskCollectionFactory.PackageCollection().ByKey([]byte(ref))
 			if err != nil {
 				if err == database.ErrNotFound {
 					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
@@ -394,7 +465,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -410,6 +481,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
 // @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
@@ -455,7 +527,7 @@ func apiReposPackagesDelete(c *gin.Context) {
 // @Tags Repos
 // @Param name path string true "Repository name"
 // @Param dir path string true "Directory of packages"
-// @Param file path string false "Filename (optional)"
+// @Param file path string true "Filename"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 200 {string} string "OK"
@@ -500,6 +572,8 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
@@ -523,7 +597,17 @@ func apiReposPackageFromDir(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -544,13 +628,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err = deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+			taskCollectionFactory.PackageCollection(), reporter, nil, taskCollectionFactory.ChecksumCollection)
 		failedFiles = append(failedFiles, failedFiles2...)
 		processedFiles = append(processedFiles, otherFiles...)
 
@@ -560,7 +644,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -613,11 +697,11 @@ type reposCopyPackageParams struct {
 // @Summary Copy Package
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
-// @Produce json
 // @Param name path string true "Destination repo"
 // @Param src path string true "Source repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -639,6 +723,8 @@ func apiReposCopyPackage(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource keys.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
 	if err != nil {
@@ -662,12 +748,26 @@ func apiReposCopyPackage(c *gin.Context) {
 	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
 
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		// Task: Create fresh factory and collections inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of both repos after lock acquired
+		dstRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(dstRepoName)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		srcRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(srcRepoName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
 		}
@@ -680,12 +780,12 @@ func apiReposCopyPackage(c *gin.Context) {
 			RemovedLines: []string{},
 		}
 
-		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
 		}
 
-		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
 		}
@@ -753,7 +853,7 @@ func apiReposCopyPackage(c *gin.Context) {
 		} else {
 			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			err = taskCollectionFactory.LocalRepoCollection().Update(dstRepo)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 			}
@@ -856,6 +956,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	resources = append(resources, sources...)
 
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()
@@ -871,8 +974,8 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
 		_, failedFiles2, err = deb.ImportChangesFiles(
 			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
-			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+			repoTemplate, context.Progress(), taskCollectionFactory.LocalRepoCollection(), taskCollectionFactory.PackageCollection(),
+			context.PackagePool(), taskCollectionFactory.ChecksumCollection, nil, query.Parse)
 		failedFiles = append(failedFiles, failedFiles2...)
 
 		if err != nil {
@@ -901,10 +1004,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
 		}
 
-                ret := reposIncludePackageFromDirResponse{
+		ret := reposIncludePackageFromDirResponse{
 			Report:      reporter,
 			FailedFiles: failedFiles,
-                    }
+		}
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
 	})
 }

api/router.go

@@ -11,13 +11,19 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog/log"
 
-	"github.com/aptly-dev/aptly/docs"
-	swaggerFiles "github.com/swaggo/files"
-	ginSwagger "github.com/swaggo/gin-swagger"
+	// _ "github.com/aptly-dev/aptly/docs" // import docs
+	// swaggerFiles "github.com/swaggo/files"
+	// ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		countPackagesByRepos()
@@ -25,21 +31,21 @@ func apiMetricsGet() gin.HandlerFunc {
 	}
 }
 
-func redirectSwagger(c *gin.Context) {
-	if c.Request.URL.Path == "/docs/index.html" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs/" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	c.Next()
-}
+// func redirectSwagger(c *gin.Context) {
+// 	if c.Request.URL.Path == "/docs/index.html" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs/" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	c.Next()
+// }
 
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
@@ -63,14 +69,14 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 	router.Use(gin.Recovery(), gin.ErrorLogger())
 
-	if c.Config().EnableSwaggerEndpoint {
-		router.GET("docs.html", func(c *gin.Context) {
-			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
-		})
-		router.Use(redirectSwagger)
-		url := ginSwagger.URL("/docs/doc.json")
-		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
-	}
+	// if c.Config().EnableSwaggerEndpoint {
+	// 	router.GET("docs.html", func(c *gin.Context) {
+	// 		c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+	// 	})
+	// 	router.Use(redirectSwagger)
+	// 	url := ginSwagger.URL("/docs/doc.json")
+	// 	router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	// }
 
 	if c.Config().EnableMetricsEndpoint {
 		MetricsCollectorRegistrar.Register(router)

api/s3.go

@@ -14,7 +14,8 @@ import (
 // @Router /api/s3 [get]
 func apiS3List(c *gin.Context) {
 	keys := []string{}
-	for k := range context.Config().S3PublishRoots {
+	s3Roots := context.Config().S3PublishRoots
+	for k := range s3Roots {
 		keys = append(keys, k)
 	}
 	c.JSON(200, keys)

api/snapshot.go

@@ -74,26 +74,33 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
 		}
 
-		err = collection.LoadComplete(repo)
+		err = taskMirrorCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -107,7 +114,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -156,6 +163,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		}
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 	var resources []string
@@ -169,37 +177,62 @@ func apiSnapshotsCreate(c *gin.Context) {
 			return
 		}
 
-		resources = append(resources, string(sources[i].ResourceKey()))
+		resources = append(resources, string(sources[i].Key()))
 	}
 
 	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		for i := range sources {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPackageCollection := taskCollectionFactory.PackageCollection()
+
+		// Fresh load of all sources after lock acquired
+		freshSources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+		for i := range b.SourceSnapshots {
+			freshSources[i], err = taskSnapshotCollection.ByName(b.SourceSnapshots[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
 		}
 
-		list := deb.NewPackageList()
+		// Merge packages from all source snapshots
+		var refList *deb.PackageRefList
+		if len(freshSources) > 0 {
+			refList = freshSources[0].RefList()
+			for i := 1; i < len(freshSources); i++ {
+				refList = refList.Merge(freshSources[i].RefList(), true, false)
+			}
+		} else {
+			refList = deb.NewPackageRefList()
+		}
 
-		// verify package refs and build package list
-		for _, ref := range b.PackageRefs {
-			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
-			if err != nil {
-				if err == database.ErrNotFound {
-					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+		// Add any explicitly specified package refs on top
+		if len(b.PackageRefs) > 0 {
+			list := deb.NewPackageList()
+			for _, ref := range b.PackageRefs {
+				p, err := taskPackageCollection.ByKey([]byte(ref))
+				if err != nil {
+					if err == database.ErrNotFound {
+						return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+					}
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+				}
+				err = list.Add(p)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 				}
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-			}
-			err = list.Add(p)
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
+			refList = refList.Merge(deb.NewPackageRefListFromPackageList(list), true, false)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+		snapshot = deb.NewSnapshotFromRefList(b.Name, freshSources, refList, b.Description)
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -217,10 +250,9 @@ type snapshotsCreateFromRepositoryParams struct {
 // @Summary Snapshot Repository
 // @Description **Create a snapshot of a repository by name**
 // @Tags Snapshots
-// @Param name path string true "Repository name"
 // @Consume json
 // @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
-// @Param name path string true "Name of the snapshot"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -241,21 +273,28 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.LoadComplete(repo)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskRepoCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskRepoCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskRepoCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -269,7 +308,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -307,6 +346,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
@@ -317,14 +357,38 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
-	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		_, err := collection.ByName(b.Name)
+	// Pre-task validation of new name if provided (skip if renaming to same name)
+	if b.Name != "" && b.Name != name {
+		_, err = collection.ByName(b.Name)
 		if err == nil {
-			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+			return
+		}
+	}
+
+	resources := []string{string(snapshot.Key())}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err = taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
+		// Fresh duplicate check inside lock
+		if b.Name != "" {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			}
+		}
+
+		// Update fresh copy
 		if b.Name != "" {
 			snapshot.Name = b.Name
 		}
@@ -333,7 +397,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		err = taskCollection.Update(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -387,9 +451,9 @@ func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
@@ -397,23 +461,37 @@ func apiSnapshotsDrop(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey())}
+	resources := []string{string(snapshot.Key())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.BySnapshot(snapshot)
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err := taskSnapshotCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Fresh checks with current collections
+		published := taskPublishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			// Using fresh collection for dependency check
+			snapshots := taskSnapshotCollection.BySnapshotSource(snapshot)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
 			}
 		}
 
-		err = snapshotCollection.Drop(snapshot)
+		err = taskSnapshotCollection.Drop(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -568,6 +646,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 
@@ -580,36 +659,47 @@ func apiSnapshotsMerge(c *gin.Context) {
 			return
 		}
 
-		resources[i] = string(sources[i].ResourceKey())
+		resources[i] = string(sources[i].Key())
 	}
 
 	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = snapshotCollection.LoadComplete(sources[0])
-		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-		}
-		result := sources[0].RefList()
-		for i := 1; i < len(sources); i++ {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load of all sources inside task
+		freshSources := make([]*deb.Snapshot, len(body.Sources))
+		for i := range body.Sources {
+			freshSources[i], err = taskSnapshotCollection.ByName(body.Sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
-			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+		}
+
+		// Merge using fresh sources
+		result := freshSources[0].RefList()
+		for i := 1; i < len(freshSources); i++ {
+			result = result.Merge(freshSources[i].RefList(), overrideMatching, false)
 		}
 
 		if latest {
 			result.FilterLatestRefs()
 		}
 
-		sourceDescription := make([]string, len(sources))
-		for i, s := range sources {
+		sourceDescription := make([]string, len(freshSources))
+		for i, s := range freshSources {
 			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+		snapshot = deb.NewSnapshotFromRefList(name, freshSources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -690,24 +780,32 @@ func apiSnapshotsPull(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	resources := []string{string(sourceSnapshot.Key()), string(toSnapshot.Key())}
 	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of snapshots after lock acquired
+		freshToSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(name)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+		freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshSourceSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		// convert snapshots to package list
-		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		toPackageList, err := deb.NewPackageListFromRefList(freshToSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		sourcePackageList, err := deb.NewPackageListFromRefList(freshSourceSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -804,10 +902,10 @@ func apiSnapshotsPull(c *gin.Context) {
 		}
 
 		// Create <destination> snapshot
-		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
-			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{freshToSnapshot, freshSourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", freshToSnapshot.Name, freshSourceSnapshot.Name, strings.Join(body.Queries, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(destinationSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}

aptly/interfaces.go

@@ -95,8 +95,8 @@ type FileSystemPublishedStorage interface {
 
 // PublishedStorageProvider is a thing that returns PublishedStorage by name
 type PublishedStorageProvider interface {
-	// GetPublishedStorage returns PublishedStorage by name
-	GetPublishedStorage(name string) PublishedStorage
+	// GetPublishedStorage returns PublishedStorage by name, or an error if the storage is not configured
+	GetPublishedStorage(name string) (PublishedStorage, error)
 }
 
 // BarType used to differentiate between different progress bars

azure/azure.go

@@ -5,35 +5,28 @@ package azure
 import (
 	"context"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net/url"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 )
 
 func isBlobNotFound(err error) bool {
-	var respErr *azcore.ResponseError
-	if errors.As(err, &respErr) {
-		return respErr.StatusCode == 404 // BlobNotFound
-	}
-	return false
+	storageError, ok := err.(azblob.StorageError)
+	return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
 }
 
 type azContext struct {
-	client    *azblob.Client
-	container string
+	container azblob.ContainerURL
 	prefix    string
 }
 
 func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
-	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
 	if err != nil {
 		return nil, err
 	}
@@ -42,14 +35,15 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) (
 		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
 	}
 
-	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
 	if err != nil {
 		return nil, err
 	}
 
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
 	result := &azContext{
-		client:    serviceClient,
-		container: container,
+		container: containerURL,
 		prefix:    prefix,
 	}
 
@@ -60,6 +54,10 @@ func (az *azContext) blobPath(path string) string {
 	return filepath.Join(az.prefix, path)
 }
 
+func (az *azContext) blobURL(path string) azblob.BlobURL {
+	return az.container.NewBlobURL(az.blobPath(path))
+}
+
 func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
 	const delimiter = "/"
 	paths = make([]string, 0, 1024)
@@ -69,33 +67,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 		prefix += delimiter
 	}
 
-	ctx := context.Background()
-	maxResults := int32(1)
-	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
-		Prefix:     &prefix,
-		MaxResults: &maxResults,
-		Include:    azblob.ListBlobsInclude{Metadata: true},
-	})
-
-	// Iterate over each page
-	for pager.More() {
-		page, err := pager.NextPage(ctx)
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := az.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
 		if err != nil {
 			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
 		}
 
-		for _, blob := range page.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, *blob.Name)
-			} else {
-				name := *blob.Name
-				paths = append(paths, name[len(prefix):])
-			}
-			b := *blob
-			md5 := b.Properties.ContentMD5
-			md5s = append(md5s, fmt.Sprintf("%x", md5))
+		marker = listBlob.NextMarker
 
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
 		}
+
 		if progress != nil {
 			time.Sleep(time.Duration(500) * time.Millisecond)
 			progress.AddBar(1)
@@ -105,27 +97,28 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 	return paths, md5s, nil
 }
 
-func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
-	uploadOptions := &azblob.UploadFileOptions{
-		BlockSize:   4 * 1024 * 1024,
-		Concurrency: 8,
+func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
 	}
 
-	path := az.blobPath(blobName)
 	if len(sourceMD5) > 0 {
 		decodedMD5, err := hex.DecodeString(sourceMD5)
 		if err != nil {
 			return err
 		}
-		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
-			BlobContentMD5: decodedMD5,
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
 		}
 	}
 
-	var err error
-	if file, ok := source.(*os.File); ok {
-		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
-	}
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob.ToBlockBlobURL(),
+		uploadOptions,
+	)
 
 	return err
 }

azure/package_pool.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/pkg/errors"
@@ -40,7 +41,10 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu
 	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
 }
 
-func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+func (pool *PackagePool) ensureChecksums(
+	poolPath string,
+	checksumStorage aptly.ChecksumStorage,
+) (*utils.ChecksumInfo, error) {
 	targetChecksums, err := checksumStorage.Get(poolPath)
 	if err != nil {
 		return nil, err
@@ -48,7 +52,8 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 
 	if targetChecksums == nil {
 		// we don't have checksums stored yet for this file
-		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		blob := pool.az.blobURL(poolPath)
+		download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 		if err != nil {
 			if isBlobNotFound(err) {
 				return nil, nil
@@ -58,7 +63,7 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 		}
 
 		targetChecksums = &utils.ChecksumInfo{}
-		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
 		if err != nil {
 			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
 		}
@@ -87,49 +92,45 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er
 }
 
 func (pool *PackagePool) Size(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	blob := pool.az.blobURL(path)
+
 	temp, err := os.CreateTemp("", "blob-download")
 	if err != nil {
-		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+		return nil, errors.Wrap(err, "error creating temporary file for blob download")
 	}
-	defer func () { _ = os.Remove(temp.Name()) }()
+	defer func() { _ = os.Remove(temp.Name()) }()
 
-	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
 	if err != nil {
-		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+		return nil, errors.Wrapf(err, "error downloading blob at %s", path)
 	}
 
 	return temp, nil
 }
 
 func (pool *PackagePool) Remove(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+		return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
 	}
 
-	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
@@ -143,6 +144,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	}
 
 	path := pool.buildPoolPath(basename, checksums)
+	blob := pool.az.blobURL(path)
 	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
 	if err != nil {
 		return "", err
@@ -158,7 +160,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	}
 	defer func() { _ = source.Close() }()
 
-	err = pool.az.putFile(path, source, checksums.MD5)
+	err = pool.az.putFile(blob, source, checksums.MD5)
 	if err != nil {
 		return "", err
 	}

azure/package_pool_test.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
@@ -50,10 +50,8 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
 
 	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
-        })
+	cnt := s.pool.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)

azure/public.go

@@ -3,22 +3,19 @@ package azure
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
 // PublishedStorage abstract file system with published files (actually hosted on Azure)
 type PublishedStorage struct {
-	// FIXME: unused ???? prefix    string
 	az        *azContext
 	pathCache map[string]map[string]string
 }
@@ -67,7 +64,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -77,15 +74,14 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 
 // RemoveDirs removes directory structure under public path
 func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
-	path = storage.az.blobPath(path)
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := filepath.Join(path, filename)
-		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
+		blob := storage.az.blobURL(filepath.Join(path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
 		}
@@ -96,8 +92,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	path = storage.az.blobPath(path)
-	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
+	blob := storage.az.blobURL(path)
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
 	}
@@ -116,8 +112,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
 	relFilePath := filepath.Join(publishedRelPath, fileName)
-	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
-	poolPath := storage.az.blobPath(prefixRelFilePath)
+	// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	// FIXME: check how to integrate publishedPrefix:
+	poolPath := storage.az.blobPath(fileName)
 
 	if storage.pathCache == nil {
 		storage.pathCache = make(map[string]map[string]string)
@@ -160,7 +157,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	}
 	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(relFilePath, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
 	if err == nil {
 		pathCache[relFilePath] = sourceMD5
 	} else {
@@ -177,60 +174,57 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 }
 
 // Internal copy or move implementation
-func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
 	const leaseDuration = 30
-	leaseID := uuid.NewString()
 
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	srcBlobClient := containerClient.NewBlobClient(src)
-	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	dstBlobURL := storage.az.blobURL(dst)
+	srcBlobURL := storage.az.blobURL(src)
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
 	}
+	defer func() { _, _ = srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) }()
+	srcBlobLeaseID := leaseResp.LeaseID()
 
-	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
-	}
-	defer func() {
-		_, _ = blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
-	}()
-
-	dstBlobClient := containerClient.NewBlobClient(dst)
-	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
-		Metadata: metadata,
-	})
-
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
 	}
 
-	copyStatus := *copyResp.CopyStatus
+	copyStatus := copyResp.CopyStatus()
 	for {
-		if copyStatus == blob.CopyStatusTypeSuccess {
+		if copyStatus == azblob.CopyStatusSuccess {
 			if move {
-				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
-					AccessConditions: &blob.AccessConditions{
-						LeaseAccessConditions: &blob.LeaseAccessConditions{
-							LeaseID: &leaseID,
-						},
-					},
-				})
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
 				return err
 			}
 			return nil
-		} else if copyStatus == blob.CopyStatusTypePending {
+		} else if copyStatus == azblob.CopyStatusPending {
 			time.Sleep(1 * time.Second)
-			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
 			if err != nil {
-				return fmt.Errorf("error getting copy progress %s", dst)
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
 			}
-			copyStatus = *getMetadata.CopyStatus
+			copyStatus = blobPropsResp.CopyStatus()
 
-			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
 			if err != nil {
-				return fmt.Errorf("error renewing source blob lease %s", src)
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
 			}
 		} else {
 			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -245,9 +239,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 // SymLink creates a copy of src file and adds link information as meta data
 func (storage *PublishedStorage) SymLink(src string, dst string) error {
-	metadata := make(map[string]*string)
-	metadata["SymLink"] = &src
-	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
 }
 
 // HardLink using symlink functionality as hard links do not exist
@@ -257,33 +249,28 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	_, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		if isBlobNotFound(err) {
 			return false, nil
 		}
-		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
 	}
-	return true, nil
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 }
 
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	props, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return "", fmt.Errorf("failed to get blob properties: %v", err)
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 	}
-
-	metadata := props.Metadata
-	if originalBlob, exists := metadata["original_blob"]; exists {
-		return *originalBlob, nil
-	}
-	return "", fmt.Errorf("error reading link %s: %v", path, err)
+	return resp.NewMetadata()["SymLink"], nil
 }

azure/public_test.go

@@ -7,11 +7,8 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-        "bytes"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 	. "gopkg.in/check.v1"
@@ -69,10 +66,8 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
-        })
+	cnt := s.storage.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -80,39 +75,41 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	cnt := s.storage.az.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	blob := s.storage.az.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, IsNil)
-	data, err := io.ReadAll(resp.Body)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := io.ReadAll(body)
 	c.Assert(err, IsNil)
 	return data
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-        serviceClient := s.storage.az.client.ServiceClient()
-        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
-        blobClient := containerClient.NewBlobClient(path)
-        _, err := blobClient.GetProperties(context.Background(), nil)
+	_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, NotNil)
-
-        storageError, ok := err.(*azcore.ResponseError)
+	storageError, ok := err.(azblob.StorageError)
 	c.Assert(ok, Equals, true)
-	c.Assert(storageError.StatusCode, Equals, 404)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
 }
 
 func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	hash := md5.Sum(data)
-        uploadOptions := &azblob.UploadStreamOptions{
-            HTTPHeaders: &blob.HTTPHeaders{
-            	BlobContentMD5: hash[:],
-            },
-        }
-        reader := bytes.NewReader(data)
-        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.az.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
 	c.Assert(err, IsNil)
 }
 
@@ -333,7 +330,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 
 	// 2nd link from pool, providing wrong path for source file
 	//
-	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
 	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)

cmd/publish.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+
+	var gpgKeys []string
+
+	// CLI args have priority over config
+	cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
+	if len(cliKeys) > 0 {
+		gpgKeys = cliKeys
+	} else if len(context.Config().GpgKeys) > 0 {
+		gpgKeys = context.Config().GpgKeys
+	}
+
+	for _, gpgKey := range gpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 
 }
 
+type gpgKeyFlag struct {
+	gpgKeys []string
+}
+
+func (k *gpgKeyFlag) Set(value string) error {
+	k.gpgKeys = append(k.gpgKeys, value)
+	return nil
+}
+
+func (k *gpgKeyFlag) Get() interface{} {
+	return k.gpgKeys
+}
+
+func (k *gpgKeyFlag) String() string {
+	return strings.Join(k.gpgKeys, ",")
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_repo.go

@@ -34,7 +34,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_snapshot.go

@@ -190,9 +190,11 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
-		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
-			localStorage.PublicPath())
+	if ps, err := context.GetPublishedStorage(storage); err == nil {
+		if localStorage, ok := ps.(aptly.FileSystemPublishedStorage); ok {
+			context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+				localStorage.PublicPath())
+		}
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
@@ -230,7 +232,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_switch.go

@@ -151,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_update.go

@@ -115,7 +115,7 @@ Example:
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/serve.go

@@ -97,7 +97,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage("")
+	if err != nil {
+		return err
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

console/progress_test.go

@@ -11,7 +11,7 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ProgressSuite struct {}
+type ProgressSuite struct{}
 
 var _ = Suite(&ProgressSuite{})
 

context/context.go

@@ -100,6 +100,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
 
 			for _, configLocation := range configLocations {
+				// FIXME: check if exists, check if readable
 				err = utils.LoadConfig(configLocation, &utils.Config)
 				if os.IsPermission(err) || os.IsNotExist(err) {
 					continue
@@ -115,7 +116,12 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
-				_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
+				defaultConfig := aptly.AptlyConf
+				if len(defaultConfig) == 0 {
+					defaultConfig = []byte("root_dir: \"\"")
+				}
+
+				_ = utils.SaveConfigRaw(homeLocation, defaultConfig)
 				err = utils.LoadConfig(homeLocation, &utils.Config)
 				if err != nil {
 					Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -406,8 +412,8 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	return context.packagePool
 }
 
-// GetPublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+// GetPublishedStorage returns instance of PublishedStorage, or an error if the storage is not configured
+func (context *AptlyContext) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	context.Lock()
 	defer context.Unlock()
 
@@ -418,14 +424,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 		} else if strings.HasPrefix(name, "filesystem:") {
 			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
-				Fatal(fmt.Errorf("published local storage %v not configured", name[11:]))
+				return nil, fmt.Errorf("published local storage %v not configured", name[11:])
 			}
 
 			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
 		} else if strings.HasPrefix(name, "s3:") {
 			params, ok := context.config().S3PublishRoots[name[3:]]
 			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+				return nil, fmt.Errorf("published S3 storage %v not configured", name[3:])
 			}
 
 			var err error
@@ -435,39 +441,39 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
 				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "swift:") {
 			params, ok := context.config().SwiftPublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Swift storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
 				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "azure:") {
 			params, ok := context.config().AzurePublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Azure storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = azure.NewPublishedStorage(
 				params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+			return nil, fmt.Errorf("unknown published storage format: %v", name)
 		}
 		context.publishedStorages[name] = publishedStorage
 	}
 
-	return publishedStorage
+	return publishedStorage, nil
 }
 
 // UploadPath builds path to upload storage

context/context_test.go

@@ -1,8 +1,6 @@
 package context
 
 import (
-	"fmt"
-	"os"
 	"reflect"
 	"testing"
 
@@ -80,10 +78,9 @@ func (s *AptlyContextSuite) SetUpTest(c *C) {
 
 func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
 	// https://github.com/aptly-dev/aptly/issues/711
-	// This will fail on account of us not having a config, so the
-	// storage never exists.
-	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
-		FatalErrorPanicMatches,
-		&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)",
-			os.Getenv("HOME"))})
+	// https://github.com/aptly-dev/aptly/issues/1477
+	// GetPublishedStorage must return an error (not panic) when the
+	// requested storage is not configured.
+	_, err := s.context.GetPublishedStorage("filesystem:fuji")
+	c.Assert(err, NotNil)
 }

database/etcddb/database_test.go

@@ -14,7 +14,7 @@ func Test(t *testing.T) {
 }
 
 type EtcDDBSuite struct {
-	db  database.Storage
+	db database.Storage
 }
 
 var _ = Suite(&EtcDDBSuite{})
@@ -133,7 +133,7 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 	v, err := s.db.Get(key)
 	c.Assert(err, IsNil)
 	c.Check(v, DeepEquals, value)
-        err = transaction.Delete(key)
+	err = transaction.Delete(key)
 	c.Assert(err, IsNil)
 
 	_, err = transaction.Get(key2)
@@ -156,4 +156,3 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 	_, err = transaction.Get(key)
 	c.Assert(err, NotNil)
 }
-

deb/format.go

@@ -288,8 +288,11 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			lastField = canonicalCase(parts[0])
 			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
-				stanza[lastField] = parts[1]
-				if parts[1] != "" {
+				// Trim trailing whitespace from the inline value so that
+				// "Package-List: " does not add empty line
+				inlineVal := strings.TrimRight(parts[1], " \t")
+				stanza[lastField] = inlineVal
+				if inlineVal != "" {
 					stanza[lastField] += "\n"
 				}
 			} else {

deb/format_test.go

@@ -128,6 +128,35 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+// Sources may contain "Package-List: " with a trailing space.
+// That trailing space must not be preserved and re-emitted
+// as a spurious blank continuation line when the stanza is written back out.
+func (s *ControlFileSuite) TestPackageListTrailingSpace(c *C) {
+	input := "Package-List: \n" +
+		" bash deb shells required arch=any\n" +
+		" bash-doc deb doc optional arch=all\n"
+
+	r := NewControlFileReader(bytes.NewBufferString(input), false, false)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(stanza["Package-List"], Equals,
+		" bash deb shells required arch=any\n"+
+			" bash-doc deb doc optional arch=all\n")
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, true, false, false)
+	c.Assert(err, IsNil)
+	c.Assert(w.Flush(), IsNil)
+
+	written := buf.String()
+	c.Assert(strings.Contains(written, "Package-List:\n \n"), Equals, false,
+		Commentf("spurious blank continuation line found in written output:\n%s", written))
+	c.Assert(strings.Contains(written, "Package-List:\n bash"), Equals, true,
+		Commentf("expected Package-List entries not found in written output:\n%s", written))
+}
+
 func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
 	s.reader = bytes.NewBufferString(installerFile)
 	r := NewControlFileReader(s.reader, false, true)

deb/list.go

@@ -598,6 +598,7 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) {
 					//
 					// when follow-all-variants is enabled, we need to try to expand anyway,
 					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					// FIXME: do not search twice
 					if result.Search(dep, false, true) != nil {
 						if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
 							options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true))

deb/local.go

@@ -168,6 +168,8 @@ func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
 
 // LoadComplete loads additional information for local repo
 func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
+	repo.packageRefs = &PackageRefList{}
+
 	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
@@ -176,7 +178,6 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 		return err
 	}
 
-	repo.packageRefs = &PackageRefList{}
 	return repo.packageRefs.Decode(encoded)
 }
 

deb/local_test.go

@@ -133,6 +133,18 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(r.String(), Equals, repo.String())
 }
 
+func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) {
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	r, err := s.collection.ByName("local1")
+	c.Assert(err, IsNil)
+
+	c.Assert(s.collection.LoadComplete(r), IsNil)
+	c.Assert(r.packageRefs, NotNil)
+	c.Assert(r.NumPackages(), Equals, 0)
+}
+
 func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Update(repo), IsNil)

deb/ppa.go

@@ -28,6 +28,11 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 		}
 	}
 
+	baseurl := config.PpaBaseURL
+	if baseurl == "" {
+		baseurl = "http://ppa.launchpad.net"
+	}
+
 	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
@@ -39,7 +44,7 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 
 	distribution = codename
 	components = []string{"main"}
-	url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID)
+	url = fmt.Sprintf("%s/%s/%s/%s", baseurl, matches[1], matches[2], distributorID)
 
 	return
 }

deb/publish.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -603,6 +604,15 @@ func (p *PublishedRepo) Key() []byte {
 	return []byte("U" + p.StoragePrefix() + ">>" + p.Distribution)
 }
 
+// PrefixPoolLockKey returns the task-queue resource key that serialises all
+// publish operations sharing the same pool directory under storagePrefix.
+// It must be held whenever a non-MultiDist publish may read or clean the
+// shared pool, to prevent concurrent cleanup runs from deleting each other's
+// files.  See docs/Resource-Locking.md for the full key-namespace table.
+func PrefixPoolLockKey(storagePrefix string) string {
+	return "P" + storagePrefix
+}
+
 // RefKey is a unique id for package reference list
 func (p *PublishedRepo) RefKey(component string) []byte {
 	return []byte("E" + p.UUID + component)
@@ -814,9 +824,12 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
-	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
+	err = publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
 	if err != nil {
 		return err
 	}
@@ -1126,7 +1139,15 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.GetSuite()
 	release["Codename"] = p.GetCodename()
-	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
+	datetimeFormat := "Mon, 2 Jan 2006 15:04:05 MST"
+
+	publishDate := time.Now().UTC()
+	if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
+		if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil {
+			publishDate = time.Unix(sec, 0).UTC()
+		}
+	}
+	release["Date"] = publishDate.Format(datetimeFormat)
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	if p.AcquireByHash {
 		release["Acquire-By-Hash"] = "yes"
@@ -1182,7 +1203,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 // It can remove prefix fully, and part of pool (for specific component)
 func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool,
 	removePoolComponents []string, progress aptly.Progress) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
 	// I. Easy: remove whole prefix (meta+packages)
 	if removePrefix {
@@ -1195,7 +1219,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 	}
 
 	// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
-	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
+	err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
 	if err != nil {
 		return err
 	}
@@ -1522,6 +1546,55 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 	return referencedFiles, nil
 }
 
+// CleanupAfterMultiDistToggle cleans up stale pool files left behind when the
+// MultiDist flag is toggled on a published repository.
+//
+//   - false→true: Publish() wrote packages into pool/<distribution>/<component>/
+//     but the old flat pool/<component>/ files were not removed because
+//     CleanupPrefixComponentFiles only scans the new MultiDist tree.
+//     A second pass with MultiDist=false cleans the legacy flat layout by
+//     reusing the existing orphan-detection logic (the repo is now MultiDist=true
+//     so it is excluded from the referenced-files scan, making its old pool
+//     entries appear orphaned).
+//
+//   - true→false: Publish() wrote packages into pool/<component>/ but the old
+//     per-distribution pool/<distribution>/<component>/ directories were not
+//     removed.  The orphan-detection approach cannot be used here because the
+//     repo's RefList still contains all packages (they just moved locations).
+//     Instead we directly remove each pool/<distribution>/<component>/ directory.
+//     This is safe because per-distribution pool dirs are exclusive to a single
+//     prefix+distribution combination — no other published repo can share them.
+func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(publishedStorageProvider aptly.PublishedStorageProvider,
+	published *PublishedRepo, prevMultiDist bool, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	if prevMultiDist == published.MultiDist {
+		return nil
+	}
+
+	if !prevMultiDist && published.MultiDist {
+		// false→true: use orphan-detection via the existing cleanup, but with
+		// MultiDist temporarily set to false so it scans the flat pool layout.
+		legacy := *published
+		legacy.MultiDist = false
+		return collection.CleanupPrefixComponentFiles(publishedStorageProvider, &legacy, cleanComponents, collectionFactory, progress)
+	}
+
+	// true→false: directly remove the per-distribution pool directories.
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
+	for _, component := range cleanComponents {
+		poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component)
+		if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil {
+			return err
+		}
+	}
+	// Remove the distribution-level pool dir if it is now empty.
+	distPoolDir := filepath.Join(published.Prefix, "pool", published.Distribution)
+	_ = publishedStorage.RemoveDirs(distPoolDir, progress)
+	return nil
+}
+
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
 	published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
@@ -1535,7 +1608,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
 	distribution := published.Distribution
 
 	rootPath := filepath.Join(prefix, "dists", distribution)
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
 
 	sort.Strings(cleanComponents)
 	publishedComponents := published.Components()

deb/publish_test.go

@@ -61,12 +61,12 @@ type FakeStorageProvider struct {
 	storages map[string]aptly.PublishedStorage
 }
 
-func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+func (p *FakeStorageProvider) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	storage, ok := p.storages[name]
 	if !ok {
-		panic(fmt.Sprintf("unknown storage: %#v", name))
+		return nil, fmt.Errorf("unknown storage: %#v", name)
 	}
-	return storage
+	return storage, nil
 }
 
 type PublishedRepoSuite struct {
@@ -433,6 +433,47 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
+func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) {
+	// Test with SOURCE_DATE_EPOCH set
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC
+	c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
+func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) {
+	// Test with invalid SOURCE_DATE_EPOCH (should fallback to current time)
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "invalid")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH)
+	c.Check(st["Date"], Not(Equals), "")
+	c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
 	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)
@@ -756,7 +797,10 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 	snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
 	_ = s.snapshotCollection.Add(snap3)
 
-	// Ensure that adding a second publish point with matching files doesn't give duplicate results.
+	// When a second publish point references the same package (snap3 is a clone of snap2,
+	// both containing p3/lonely-strangers), listReferencedFilesByComponent deduplicates by
+	// package ref so the file appears only once.  StrSlicesSubstract handles a single entry
+	// correctly, so no duplicate is needed for cleanup safety.
 	repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(s.collection.Add(repo3), IsNil)
@@ -771,7 +815,9 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 			"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			"a/alien-arena/mars-invaders_7.40-2_i386.deb",
 		},
-		"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
+		"main": {
+			"a/alien-arena/lonely-strangers_7.40-2_i386.deb",
+		},
 	})
 }
 

deb/reflist.go

@@ -79,6 +79,9 @@ func (l *PackageRefList) Decode(input []byte) error {
 
 // ForEach calls handler for each package ref in list
 func (l *PackageRefList) ForEach(handler func([]byte) error) error {
+	if l == nil {
+		return nil
+	}
 	var err error
 	for _, p := range l.Refs {
 		err = handler(p)

deb/reflist_test.go

@@ -65,7 +65,7 @@ func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
 	list, err := NewPackageListFromRefList(reflist, coll, nil)
 	c.Assert(err, IsNil)
 	c.Check(list.Len(), Equals, 4)
-        c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
+	c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
 
 	list, err = NewPackageListFromRefList(nil, coll, nil)
 	c.Assert(err, IsNil)
@@ -130,6 +130,17 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestForEachNilList(c *C) {
+	var l *PackageRefList
+	called := false
+	err := l.ForEach(func([]byte) error {
+		called = true
+		return nil
+	})
+	c.Assert(err, IsNil)
+	c.Assert(called, Equals, false)
+}
+
 func (s *PackageRefListSuite) TestHas(c *C) {
 	_ = s.list.Add(s.p1)
 	_ = s.list.Add(s.p3)

deb/remote.go

@@ -574,7 +574,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					if progress != nil {
 						progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
 					}
-				} else if err != nil {
+				} else {
 					return err
 				}
 			}

deb/snapshot.go

@@ -125,12 +125,6 @@ func (s *Snapshot) Key() []byte {
 	return []byte("S" + s.UUID)
 }
 
-// ResourceKey is a unique identifier of the resource
-// this snapshot uses. Instead of uuid it uses name
-// which needs to be unique as well.
-func (s *Snapshot) ResourceKey() []byte {
-	return []byte("S" + s.Name)
-}
 
 // RefKey is a unique id for package reference list
 func (s *Snapshot) RefKey() []byte {

deb/snapshot_bench_test.go

@@ -31,8 +31,7 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		collection = NewSnapshotCollection(db)
 
-
-                _ = collection.ForEach(func(s *Snapshot) error {
+		_ = collection.ForEach(func(s *Snapshot) error {
 			return nil
 		})
 	}

deb/version.go

@@ -30,16 +30,16 @@ func CompareVersions(ver1, ver2 string) int {
 
 // parseVersions breaks down full version to components (possibly empty)
 func parseVersion(ver string) (epoch, upstream, debian string) {
-	i := strings.LastIndex(ver, "-")
-	if i != -1 {
-		debian, ver = ver[i+1:], ver[:i]
-	}
-
-	i = strings.Index(ver, ":")
+	i := strings.Index(ver, ":")
 	if i != -1 {
 		epoch, ver = ver[:i], ver[i+1:]
 	}
 
+	i = strings.Index(ver, "-")
+	if i != -1 {
+		debian, ver = ver[i+1:], ver[:i]
+	}
+
 	upstream = ver
 
 	return
@@ -50,7 +50,7 @@ func compareLexicographic(s1, s2 string) int {
 	i := 0
 	l1, l2 := len(s1), len(s2)
 
-	for !(i == l1 && i == l2) {  // break if s1 equal to s2
+	for !(i == l1 && i == l2) { // break if s1 equal to s2
 
 		if i == l2 {
 			// s1 is longer than s2

deb/version_test.go

@@ -20,10 +20,10 @@ func (s *VersionSuite) TestParseVersion(c *C) {
 	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3.4", "1"})
 
 	e, u, d = parseVersion("1.3-pre4-1")
-	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3-pre4", "1"})
+	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3", "pre4-1"})
 
 	e, u, d = parseVersion("4:1.3-pre4-1")
-	c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3-pre4", "1"})
+	c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3", "pre4-1"})
 }
 
 func (s *VersionSuite) TestCompareLexicographic(c *C) {
@@ -100,6 +100,7 @@ func (s *VersionSuite) TestCompareVersions(c *C) {
 	c.Check(CompareVersions("1.0-133-avc", "1.0"), Equals, 1)
 
 	c.Check(CompareVersions("5.2.0.3", "5.2.0.283"), Equals, -1)
+	c.Check(CompareVersions("4.3.5a", "4.3.5-rc3-1"), Equals, 1)
 }
 
 func (s *VersionSuite) TestParseDependency(c *C) {

debian/aptly.conf

@@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu
 # Codename for short PPA url expansion
 ppa_codename: ""
 
+# PPA Base URL (default: launchpad)
+# # ppa_baseurl: http://ppa.launchpad.net
+
 
 # Aptly Server
 ###############

docs/Files.md

@@ -2,7 +2,7 @@
 <div>
 
 In order to add debian package files to a local repository, files are first uploaded to a temporary directory.
-Then the directory (or a specific file within) is added to a repository. After adding to a repositorty, the directory resp. files are removed bt default.
+Then the directory (or a specific file within) is added to a repository. After adding to a repository, the directory resp. files are removed bt default.
 
 All uploaded files are stored under `<rootDir>/upload/<tempdir>` directory.
 

docs/Packages.md

@@ -1,5 +1,5 @@
 # Search Package Collection
 <div>
-Perform operations on the whole collection of packages in apty database.
+Perform operations on the whole collection of packages in aptly database.
 </div>
 

docs/Publish.md

@@ -11,11 +11,30 @@ Repositories can be published to local directories, Amazon S3 buckets, Azure or
 
 GPG key is required to sign any published repository. The key pari should be generated before publishing.
 
-Publiс part of the key should be exported from your keyring using `gpg --export --armor` and imported on the system which uses a published repository.
+Public part of the key should be exported from your keyring using `gpg --export --armor` and imported on the system which uses a published repository.
+
+* Multiple signing keys can be defined in aptly.conf using the gpgKeys array:
+```
+"gpgKeys": [
+    "KEY_ID_x",
+    "KEY_ID_y"
+]
+```
+
+* It is also possible to pass multiple keys via the CLI using the repeatable `--gpg-key` flag:
+```
+aptly publish repo my-repo --gpg-key=KEY_ID_a --gpg-key=KEY_ID_b
+```
+* When using the REST API, the `gpgKey` parameter supports a comma-separated list of key IDs:
+```
+"gpgKey": "KEY_ID_a,KEY_ID_b"
+```
+* If `--gpg-key` is specified on the command line, or `gpgKey` is provided via the REST API, it takes precedence over any gpgKeys configuration in aptly.conf.
+* With multi-key support, aptly will sign all Release files (both clearsigned and detached signatures) with each provided key, ensuring a smooth key rotation process while maintaining compatibility for existing clients.
 
 #### Parameters
 
 Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`.  `:distribution` is distribution name, while `:prefix` is `[<storage>:]<prefix>` (storage is optional, it defaults to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores
-should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambigious in URLs.
+should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambiguous in URLs.
 
 </div>

docs/Repos.md

@@ -1,6 +1,6 @@
 # Manage Local Repositories
 <div>
-A local repository is a collection of versionned packages (usually custom packages created internally).
+A local repository is a collection of versioned packages (usually custom packages created internally).
 
 Packages can be added, removed, moved or copied between repos.
 

files/package_pool.go

@@ -241,7 +241,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 		return "", err
 	}
 	defer func() {
-            _ = source.Close()
+		_ = source.Close()
 	}()
 
 	sourceInfo, err := source.Stat()

files/public.go

@@ -15,6 +15,10 @@ import (
 	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 // PublishedStorage abstract file system with public dirs (published repos)
 type PublishedStorage struct {
 	rootPath     string
@@ -99,7 +103,17 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}()
 
 	_, err = io.Copy(f, source)
-	return err
+	if err != nil {
+		return err
+	}
+
+	// Sync to ensure all data is written to disk and catch ENOSPC errors
+	err = syncFile(f)
+	if err != nil {
+		return fmt.Errorf("error syncing file %s: %s", path, err)
+	}
+
+	return nil
 }
 
 // Remove removes single file under public path
@@ -136,6 +150,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 
 	baseName := filepath.Base(fileName)
 	poolPath := filepath.Join(storage.rootPath, publishedPrefix, publishedRelPath, filepath.Dir(fileName))
+	destinationPath := filepath.Join(poolPath, baseName)
 
 	var localSourcePool aptly.LocalPackagePool
 	if storage.linkMethod != LinkMethodCopy {
@@ -154,7 +169,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 
 	var dstStat os.FileInfo
 
-	dstStat, err = os.Stat(filepath.Join(poolPath, baseName))
+	dstStat, err = os.Stat(destinationPath)
 	if err == nil {
 		// already exists, check source file
 
@@ -173,7 +188,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 			} else {
 				// if source and destination have the same checksums, no need to copy
 				var dstMD5 string
-				dstMD5, err = utils.MD5ChecksumForFile(filepath.Join(poolPath, baseName))
+				dstMD5, err = utils.MD5ChecksumForFile(destinationPath)
 
 				if err != nil {
 					return err
@@ -204,11 +219,11 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 
 		// source and destination have different inodes, if !forced, this is fatal error
 		if !force {
-			return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName))
+			return fmt.Errorf("error linking file to %s: file already exists and is different", destinationPath)
 		}
 
 		// forced, so remove destination
-		err = os.Remove(filepath.Join(poolPath, baseName))
+		err = os.Remove(destinationPath)
 		if err != nil {
 			return err
 		}
@@ -223,7 +238,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 		}
 
 		var dst *os.File
-		dst, err = os.Create(filepath.Join(poolPath, baseName))
+		dst, err = os.Create(destinationPath)
 		if err != nil {
 			_ = r.Close()
 			return err
@@ -242,11 +257,18 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 			return err
 		}
 
+		// Sync to ensure all data is written to disk and catch ENOSPC errors
+		err = syncFile(dst)
+		if err != nil {
+			_ = dst.Close()
+			return fmt.Errorf("error syncing file %s: %s", destinationPath, err)
+		}
+
 		err = dst.Close()
 	} else if storage.linkMethod == LinkMethodSymLink {
-		err = localSourcePool.Symlink(sourcePath, filepath.Join(poolPath, baseName))
+		err = localSourcePool.Symlink(sourcePath, destinationPath)
 	} else {
-		err = localSourcePool.Link(sourcePath, filepath.Join(poolPath, baseName))
+		err = localSourcePool.Link(sourcePath, destinationPath)
 	}
 
 	return err

files/public_test.go

@@ -1,8 +1,14 @@
 package files
 
 import (
+	"bytes"
+	"errors"
+	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
+	"strings"
 	"syscall"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -11,6 +17,77 @@ import (
 	. "gopkg.in/check.v1"
 )
 
+type fakeProgress struct{ bytes.Buffer }
+
+func (p *fakeProgress) Start()    {}
+func (p *fakeProgress) Shutdown() {}
+func (p *fakeProgress) Flush()    {}
+func (p *fakeProgress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
+}
+func (p *fakeProgress) ShutdownBar()     {}
+func (p *fakeProgress) AddBar(count int) {}
+func (p *fakeProgress) SetBar(count int) {}
+func (p *fakeProgress) Printf(msg string, a ...interface{}) {
+}
+func (p *fakeProgress) ColoredPrintf(msg string, a ...interface{}) {
+}
+func (p *fakeProgress) PrintfStdErr(msg string, a ...interface{}) {
+}
+
+type fakeRSC struct {
+	*bytes.Reader
+	closeErr error
+}
+
+func (r *fakeRSC) Close() error { return r.closeErr }
+
+type fakePool struct {
+	sizeErr error
+	openFn  func(string) (aptly.ReadSeekerCloser, error)
+}
+
+type fakeLocalPool struct {
+	fakePool
+	statErr error
+}
+
+func (p *fakeLocalPool) Stat(path string) (os.FileInfo, error) { return nil, p.statErr }
+func (p *fakeLocalPool) GenerateTempPath(filename string) (string, error) {
+	return "", nil
+}
+func (p *fakeLocalPool) Link(path, dstPath string) error    { return nil }
+func (p *fakeLocalPool) Symlink(path, dstPath string) error { return nil }
+func (p *fakeLocalPool) FullPath(path string) string        { return path }
+
+func (p *fakePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
+	return "", false, nil
+}
+
+func (p *fakePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage aptly.ChecksumStorage) (string, error) {
+	return "", nil
+}
+
+func (p *fakePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
+	return "", nil
+}
+
+func (p *fakePool) Size(path string) (int64, error) {
+	if p.sizeErr != nil {
+		return 0, p.sizeErr
+	}
+	return int64(len(path)), nil
+}
+
+func (p *fakePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	if p.openFn != nil {
+		return p.openFn(path)
+	}
+	return nil, io.EOF
+}
+
+func (p *fakePool) FilepathList(progress aptly.Progress) ([]string, error) { return nil, nil }
+func (p *fakePool) Remove(path string) (int64, error)                      { return 0, nil }
+
 type PublishedStorageSuite struct {
 	root            string
 	storage         *PublishedStorage
@@ -69,6 +146,14 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	c.Assert(err, IsNil)
 }
 
+func (s *PublishedStorageSuite) TestPutFileReturnsErrorIfSourceMissing(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", filepath.Join(s.root, "no-such-file"))
+	c.Assert(err, NotNil)
+}
+
 func (s *PublishedStorageSuite) TestFilelist(c *C) {
 	err := s.storage.MkDir("ppa/pool/main/a/ab/")
 	c.Assert(err, IsNil)
@@ -134,6 +219,11 @@ func (s *PublishedStorageSuite) TestSymLink(c *C) {
 	c.Assert(linkTarget, Equals, "ppa/dists/squeeze/Release")
 }
 
+func (s *PublishedStorageSuite) TestReadLinkReturnsErrorOnMissingPath(c *C) {
+	_, err := s.storage.ReadLink("does/not/exist")
+	c.Assert(err, NotNil)
+}
+
 func (s *PublishedStorageSuite) TestHardLink(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
@@ -163,6 +253,18 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) {
 	c.Assert(os.IsNotExist(err), Equals, true)
 }
 
+func (s *PublishedStorageSuite) TestRemoveDirsWithProgress(c *C) {
+	err := s.storage.MkDir("ppa/dists/squeeze/")
+	c.Assert(err, IsNil)
+
+	err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null")
+	c.Assert(err, IsNil)
+
+	p := &fakeProgress{}
+	err = s.storage.RemoveDirs("ppa/dists/", p)
+	c.Assert(err, IsNil)
+}
+
 func (s *PublishedStorageSuite) TestRemove(c *C) {
 	err := s.storage.MkDir("ppa/dists/squeeze/")
 	c.Assert(err, IsNil)
@@ -337,3 +439,277 @@ func (s *PublishedStorageSuite) TestRootRemove(c *C) {
 	dirStorage := NewPublishedStorage(pwd, "", "")
 	c.Assert(func() { _ = dirStorage.RemoveDirs("", nil) }, PanicMatches, "trying to remove the root directory")
 }
+
+// DiskFullSuite uses a loopback mount; requires Linux + root.
+
+type DiskFullSuite struct {
+	root string
+}
+
+var _ = Suite(&DiskFullSuite{})
+
+func (s *DiskFullSuite) SetUpTest(c *C) {
+	if runtime.GOOS != "linux" {
+		c.Skip("disk full tests only run on Linux")
+	}
+
+	s.root = c.MkDir()
+}
+
+func (s *DiskFullSuite) TestPutFileOutOfSpace(c *C) {
+	mountPoint := "/smallfs"
+	if os.Geteuid() == 0 {
+		mountPoint = filepath.Join(s.root, "smallfs")
+		err := os.MkdirAll(mountPoint, 0777)
+		c.Assert(err, IsNil)
+		fsImage := filepath.Join(s.root, "small.img")
+		cmd := exec.Command("dd", "if=/dev/zero", "of="+fsImage, "bs=1M", "count=1")
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+		cmd = exec.Command("mkfs.ext4", "-F", fsImage)
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+		cmd = exec.Command("mount", "-o", "loop", fsImage, mountPoint)
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+		defer func() {
+			_ = exec.Command("umount", mountPoint).Run()
+		}()
+	}
+
+	storage := NewPublishedStorage(mountPoint, "", "")
+	largeFile := filepath.Join(s.root, "largefile")
+	cmd := exec.Command("dd", "if=/dev/zero", "of="+largeFile, "bs=1M", "count=2")
+	err := cmd.Run()
+	c.Assert(err, IsNil)
+
+	err = storage.PutFile("testfile", largeFile)
+	c.Assert(err, NotNil)
+	c.Check(strings.Contains(err.Error(), "no space left on device") ||
+		strings.Contains(err.Error(), "sync"), Equals, true,
+		Commentf("Expected disk full error, got: %v", err))
+}
+
+func (s *DiskFullSuite) TestLinkFromPoolCopyOutOfSpace(c *C) {
+	mountPoint := "/smallfs"
+	if os.Geteuid() == 0 {
+		mountPoint = filepath.Join(s.root, "smallfs")
+		err := os.MkdirAll(mountPoint, 0777)
+		c.Assert(err, IsNil)
+		fsImage := filepath.Join(s.root, "small.img")
+
+		cmd := exec.Command("dd", "if=/dev/zero", "of="+fsImage, "bs=1M", "count=1")
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+
+		cmd = exec.Command("mkfs.ext4", "-F", fsImage)
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+
+		cmd = exec.Command("mount", "-o", "loop", fsImage, mountPoint)
+		err = cmd.Run()
+		c.Assert(err, IsNil)
+		defer func() {
+			_ = exec.Command("umount", mountPoint).Run()
+		}()
+	}
+
+	storage := NewPublishedStorage(mountPoint, "copy", "")
+
+	poolPath := filepath.Join(s.root, "pool")
+	pool := NewPackagePool(poolPath, false)
+	cs := NewMockChecksumStorage()
+
+	largeFile := filepath.Join(s.root, "package.deb")
+	cmd := exec.Command("dd", "if=/dev/zero", "of="+largeFile, "bs=1M", "count=2")
+	err := cmd.Run()
+	c.Assert(err, IsNil)
+
+	sourceChecksum, err := utils.ChecksumsForFile(largeFile)
+	c.Assert(err, IsNil)
+
+	srcPoolPath, err := pool.Import(largeFile, "package.deb",
+		&utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs)
+	c.Assert(err, IsNil)
+
+	err = storage.LinkFromPool("", "pool/main/p/package", "package.deb",
+		pool, srcPoolPath, sourceChecksum, false)
+	c.Assert(err, NotNil)
+	c.Check(strings.Contains(err.Error(), "no space left on device") ||
+		strings.Contains(err.Error(), "sync"), Equals, true,
+		Commentf("Expected disk full error, got: %v", err))
+}
+
+type DiskFullNoRootSuite struct {
+	root string
+}
+
+var _ = Suite(&DiskFullNoRootSuite{})
+
+func (s *DiskFullNoRootSuite) SetUpTest(c *C) {
+	s.root = c.MkDir()
+}
+
+func (s *DiskFullNoRootSuite) TestSyncIsCalled(c *C) {
+	storage := NewPublishedStorage(s.root, "", "")
+	sourceFile := filepath.Join(s.root, "source.txt")
+	err := os.WriteFile(sourceFile, []byte("test content"), 0644)
+	c.Assert(err, IsNil)
+	err = storage.PutFile("dest.txt", sourceFile)
+	c.Assert(err, IsNil)
+	content, err := os.ReadFile(filepath.Join(s.root, "dest.txt"))
+	c.Assert(err, IsNil)
+	c.Check(string(content), Equals, "test content")
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopySyncIsCalled(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "")
+	poolPath := filepath.Join(s.root, "pool")
+	pool := NewPackagePool(poolPath, false)
+	cs := NewMockChecksumStorage()
+
+	pkgFile := filepath.Join(s.root, "package.deb")
+	err := os.WriteFile(pkgFile, []byte("package content"), 0644)
+	c.Assert(err, IsNil)
+
+	sourceChecksum, err := utils.ChecksumsForFile(pkgFile)
+	c.Assert(err, IsNil)
+
+	srcPoolPath, err := pool.Import(pkgFile, "package.deb",
+		&utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs)
+	c.Assert(err, IsNil)
+
+	err = storage.LinkFromPool("", "pool/main/p/package", "package.deb",
+		pool, srcPoolPath, sourceChecksum, false)
+	c.Assert(err, IsNil)
+
+	destPath := filepath.Join(s.root, "pool/main/p/package/package.deb")
+	content, err := os.ReadFile(destPath)
+	c.Assert(err, IsNil)
+	c.Check(string(content), Equals, "package content")
+}
+
+func (s *DiskFullNoRootSuite) TestPutFileSyncErrorIsReturned(c *C) {
+	storage := NewPublishedStorage(s.root, "", "")
+
+	sourceFile := filepath.Join(s.root, "source-syncfail.txt")
+	err := os.WriteFile(sourceFile, []byte("test content"), 0644)
+	c.Assert(err, IsNil)
+
+	oldSyncFile := syncFile
+	syncFile = func(_ *os.File) error { return syscall.ENOSPC }
+	defer func() { syncFile = oldSyncFile }()
+
+	err = storage.PutFile("dest-syncfail.txt", sourceFile)
+	c.Assert(err, NotNil)
+	c.Check(strings.Contains(err.Error(), "error syncing file"), Equals, true)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopySyncErrorIsReturned(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "")
+	poolPath := filepath.Join(s.root, "pool")
+	pool := NewPackagePool(poolPath, false)
+	cs := NewMockChecksumStorage()
+
+	pkgFile := filepath.Join(s.root, "package-syncfail.deb")
+	err := os.WriteFile(pkgFile, []byte("package content"), 0644)
+	c.Assert(err, IsNil)
+
+	sourceChecksum, err := utils.ChecksumsForFile(pkgFile)
+	c.Assert(err, IsNil)
+
+	srcPoolPath, err := pool.Import(pkgFile, "package-syncfail.deb",
+		&utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs)
+	c.Assert(err, IsNil)
+
+	oldSyncFile := syncFile
+	syncFile = func(_ *os.File) error { return syscall.ENOSPC }
+	defer func() { syncFile = oldSyncFile }()
+
+	err = storage.LinkFromPool("", "pool/main/p/package", "package-syncfail.deb",
+		pool, srcPoolPath, sourceChecksum, false)
+	c.Assert(err, NotNil)
+	c.Check(strings.Contains(err.Error(), "error syncing file"), Equals, true)
+}
+
+func (s *DiskFullNoRootSuite) TestPutFileFailsIfDestinationDirMissing(c *C) {
+	storage := NewPublishedStorage(s.root, "", "")
+
+	sourceFile := filepath.Join(s.root, "src.txt")
+	err := os.WriteFile(sourceFile, []byte("x"), 0644)
+	c.Assert(err, IsNil)
+
+	err = storage.PutFile("missingdir/dest.txt", sourceFile)
+	c.Assert(err, NotNil)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolRejectsNonLocalPoolForHardlink(c *C) {
+	storage := NewPublishedStorage(s.root, "", "")
+	pool := &fakePool{}
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+	c.Check(strings.Contains(err.Error(), "cannot link"), Equals, true)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfOpenFails(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "")
+	pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) { return nil, io.ErrUnexpectedEOF }}
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfReaderCloseFails(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "")
+
+	pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) {
+		return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: io.ErrClosedPipe}, nil
+	}}
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+	c.Check(err, Equals, io.ErrClosedPipe)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfSizeFailsWhenDestExists(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "size")
+	pool := &fakePool{sizeErr: io.ErrUnexpectedEOF, openFn: func(string) (aptly.ReadSeekerCloser, error) {
+		return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: nil}, nil
+	}}
+
+	destDir := filepath.Join(s.root, "pool/main/p/pkg")
+	c.Assert(os.MkdirAll(destDir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(destDir, "x.deb"), []byte("old"), 0644), IsNil)
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+	c.Check(err, Equals, io.ErrUnexpectedEOF)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyChecksumReturnsErrorIfDstMD5Fails(c *C) {
+	storage := NewPublishedStorage(s.root, "copy", "")
+	pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) {
+		return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: nil}, nil
+	}}
+
+	// Make destinationPath a directory so MD5ChecksumForFile fails.
+	destDir := filepath.Join(s.root, "pool/main/p/pkg")
+	c.Assert(os.MkdirAll(destDir, 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(destDir, "x.deb"), 0777), IsNil)
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+}
+
+func (s *DiskFullNoRootSuite) TestLinkFromPoolHardlinkReturnsErrorIfStatFailsWhenDestExists(c *C) {
+	storage := NewPublishedStorage(c.MkDir(), "hardlink", "")
+	pool := &fakeLocalPool{statErr: errors.New("stat failed")}
+
+	destDir := filepath.Join(storage.rootPath, "pool", "main", "p", "pkg")
+	c.Assert(os.MkdirAll(destDir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(destDir, "x.deb"), []byte("x"), 0644), IsNil)
+
+	err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false)
+	c.Assert(err, NotNil)
+}

go.mod

@@ -41,7 +41,7 @@ require (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
@@ -87,6 +87,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -115,8 +116,7 @@ require (
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1
+	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/ProtonMail/go-crypto v1.0.0
 	github.com/aws/aws-sdk-go-v2 v1.32.5
 	github.com/aws/aws-sdk-go-v2/config v1.28.5
@@ -124,8 +124,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1
 	github.com/aws/smithy-go v1.22.1
 	github.com/google/uuid v1.6.0
-	github.com/swaggo/files v1.0.1
-	github.com/swaggo/gin-swagger v1.6.0
 	github.com/swaggo/swag v1.16.3
 	go.etcd.io/etcd/client/v3 v3.5.15
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -1,17 +1,20 @@
 github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI=
 github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
-github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
-github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
-github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
-github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
+github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U=
+github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=
+github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk=
+github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58=
+github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
+github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
+github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
+github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
+github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
+github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
+github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 h1:jbGlDKdzAZ92NzK65hUP98ri0/r50vVVvmZsFP/nIqo=
 github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI=
 github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
@@ -91,14 +94,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
 github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk=
+github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4=
-github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
@@ -127,8 +130,6 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
-github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -150,6 +151,7 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
@@ -197,6 +199,8 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
+github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
@@ -235,8 +239,6 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
 github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
 github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
-github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -284,10 +286,6 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE=
-github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg=
-github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M=
-github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo=
 github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg=
 github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk=
 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
@@ -319,6 +317,8 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
@@ -333,18 +333,19 @@ golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=
 golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
@@ -362,6 +363,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -393,6 +395,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
 golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=

http/download.go

@@ -240,7 +240,7 @@ func (downloader *downloaderImpl) download(req *http.Request, url, destination s
 	}
 	if resp.Body != nil {
 		defer func() {
-                    _ = resp.Body.Close()
+			_ = resp.Body.Close()
 		}()
 	}
 

http/download_go16.go

@@ -1,3 +1,4 @@
+//go:build !go1.7
 // +build !go1.7
 
 package http

http/grab.go

@@ -49,9 +49,9 @@ func (d *GrabDownloader) Download(ctx context.Context, url string, destination s
 
 func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
 	maxTries := d.maxTries
-        // FIXME: const delayMax = time.Duration(5 * time.Minute)
+	// FIXME: const delayMax = time.Duration(5 * time.Minute)
 	delay := time.Duration(1 * time.Second)
-        // FIXME: const delayMultiplier = 2
+	// FIXME: const delayMultiplier = 2
 	err := fmt.Errorf("no tries available")
 	for maxTries > 0 {
 		err = d.download(ctx, url, destination, expected, ignoreMismatch)
@@ -133,17 +133,17 @@ func (d *GrabDownloader) download(_ context.Context, url string, destination str
 
 	resp := d.client.Do(req)
 
-        <-resp.Done
+	<-resp.Done
 	// download is complete
 
-// Loop:
-// 	for {
-// 		select {
-// 		case <-resp.Done:
-// 			// download is complete
-// 			break Loop
-// 		}
-// 	}
+	// Loop:
+	// 	for {
+	// 		select {
+	// 		case <-resp.Done:
+	// 			// download is complete
+	// 			break Loop
+	// 		}
+	// 	}
 	err = resp.Err()
 	if err != nil && err == grab.ErrBadChecksum && ignoreMismatch {
 		fmt.Printf("Ignoring checksum mismatch for %s\n", url)

man/aptly.1

@@ -111,8 +111,9 @@ The legacy json configuration is still supported (and also supports comments):
   // Enable metrics for Prometheus client
   "enableMetricsEndpoint": false,
 
+  // Not implemented in this version\.
   // Enable API documentation on /docs
-  "enableSwaggerEndpoint": false,
+  //"enableSwaggerEndpoint": false,
 
   // OBSOLETE: use via url param ?_async=true
   "AsyncAPI": false,
@@ -2452,6 +2453,9 @@ show yaml config
 .SH "ENVIRONMENT"
 If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\.
 .
+.P
+If environment variable \fBSOURCE_DATE_EPOCH\fR is set to a Unix timestamp, \fBaptly\fR would use that timestamp for the \fBDate\fR and \fBValid\-Until\fR fields in the \fBRelease\fR file when publishing\. This enables reproducible builds as specified by \fIhttps://reproducible\-builds\.org/specs/source\-date\-epoch/\fR\.
+.
 .SH "RETURN VALUES"
 \fBaptly\fR exists with:
 .

man/aptly.1.ronn.tmpl

@@ -100,8 +100,9 @@ The legacy json configuration is still supported (and also supports comments):
       // Enable metrics for Prometheus client
       "enableMetricsEndpoint": false,
 
+      // Not implemented in this version.
       // Enable API documentation on /docs
-      "enableSwaggerEndpoint": false,
+      //"enableSwaggerEndpoint": false,
 
       // OBSOLETE: use via url param ?_async=true
       "AsyncAPI": false,
@@ -533,6 +534,11 @@ For example, default aptly display format could be presented with the following
 If environment variable `HTTP_PROXY` is set `aptly` would use its value
 to proxy all HTTP requests.
 
+If environment variable `SOURCE_DATE_EPOCH` is set to a Unix timestamp,
+`aptly` would use that timestamp for the `Date` and `Valid-Until` fields
+in the `Release` file when publishing. This enables reproducible builds
+as specified by https://reproducible-builds.org/specs/source-date-epoch/.
+
 ## RETURN VALUES
 
 `aptly` exists with:

pgp/gnupg.go

@@ -22,7 +22,7 @@ var (
 type GpgSigner struct {
 	gpg                        string
 	version                    GPGVersion
-	keyRef                     string
+	keyRefs                    []string
 	keyring, secretKeyring     string
 	passphrase, passphraseFile string
 	batch                      bool
@@ -35,7 +35,14 @@ func (g *GpgSigner) SetBatch(batch bool) {
 
 // SetKey sets key ID to use when signing files
 func (g *GpgSigner) SetKey(keyRef string) {
-	g.keyRef = keyRef
+    keyRef = strings.TrimSpace(keyRef)
+    if keyRef != "" {
+        if g.keyRefs == nil {
+            g.keyRefs = []string{keyRef}
+        } else {
+            g.keyRefs = append(g.keyRefs, keyRef)
+        }
+    }
 }
 
 // SetKeyRing allows to set custom keyring and secretkeyring
@@ -57,8 +64,8 @@ func (g *GpgSigner) gpgArgs() []string {
 		args = append(args, "--secret-keyring", g.secretKeyring)
 	}
 
-	if g.keyRef != "" {
-		args = append(args, "-u", g.keyRef)
+	for _, k := range g.keyRefs {
+		args = append(args, "-u", k)
 	}
 
 	if g.passphrase != "" || g.passphraseFile != "" {

s3/public.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
@@ -51,6 +52,7 @@ type PublishedStorage struct {
 	plusWorkaround   bool
 	disableMultiDel  bool
 	pathCache        map[string]string
+	pathCacheMutex   sync.RWMutex
 
 	// True if the bucket encrypts objects by default.
 	encryptByDefault bool
@@ -251,7 +253,9 @@ func (storage *PublishedStorage) Remove(path string) error {
 		_ = storage.Remove(strings.Replace(path, "+", " ", -1))
 	}
 
+	storage.pathCacheMutex.Lock()
 	delete(storage.pathCache, path)
+	storage.pathCacheMutex.Unlock()
 
 	return nil
 }
@@ -280,7 +284,9 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 			if err != nil {
 				return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err)
 			}
+			storage.pathCacheMutex.Lock()
 			delete(storage.pathCache, filepath.Join(path, filelist[i]))
+			storage.pathCacheMutex.Unlock()
 		}
 	} else {
 		numParts := (len(filelist) + page - 1) / page
@@ -313,9 +319,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 			if err != nil {
 				return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err)
 			}
+			storage.pathCacheMutex.Lock()
 			for i := range part {
 				delete(storage.pathCache, filepath.Join(path, part[i]))
 			}
+			storage.pathCacheMutex.Unlock()
 		}
 	}
 
@@ -337,20 +345,31 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	relPath := filepath.Join(publishedDirectory, fileName)
 	poolPath := filepath.Join(storage.prefix, relPath)
 
-	if storage.pathCache == nil {
-		paths, md5s, err := storage.internalFilelist(filepath.Join(publishedPrefix, "pool"), true)
-		if err != nil {
-			return errors.Wrap(err, "error caching paths under prefix")
-		}
+	storage.pathCacheMutex.RLock()
+	cacheNil := storage.pathCache == nil
+	storage.pathCacheMutex.RUnlock()
 
-		storage.pathCache = make(map[string]string, len(paths))
+	if cacheNil {
+		storage.pathCacheMutex.Lock()
+		if storage.pathCache == nil {
+			paths, md5s, err := storage.internalFilelist(filepath.Join(publishedPrefix, "pool"), true)
+			if err != nil {
+				storage.pathCacheMutex.Unlock()
+				return errors.Wrap(err, "error caching paths under prefix")
+			}
 
-		for i := range paths {
-			storage.pathCache[filepath.Join("pool", paths[i])] = md5s[i]
+			storage.pathCache = make(map[string]string, len(paths))
+
+			for i := range paths {
+				storage.pathCache[filepath.Join(publishedPrefix, "pool", paths[i])] = md5s[i]
+			}
 		}
+		storage.pathCacheMutex.Unlock()
 	}
 
+	storage.pathCacheMutex.RLock()
 	destinationMD5, exists := storage.pathCache[relPath]
+	storage.pathCacheMutex.RUnlock()
 	sourceMD5 := sourceChecksums.MD5
 
 	if exists {
@@ -367,7 +386,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 				err = errors.Wrap(err, fmt.Sprintf("error verifying MD5 for %s: %s", storage, poolPath))
 				return err
 			}
+			storage.pathCacheMutex.Lock()
 			storage.pathCache[relPath] = destinationMD5
+			storage.pathCacheMutex.Unlock()
 		}
 
 		if destinationMD5 == sourceMD5 {
@@ -388,7 +409,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	log.Debug().Msgf("S3: LinkFromPool '%s'", relPath)
 	err = storage.putFile(relPath, source, sourceMD5)
 	if err == nil {
+		storage.pathCacheMutex.Lock()
 		storage.pathCache[relPath] = sourceMD5
+		storage.pathCacheMutex.Unlock()
 	} else {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
 	}

s3/server_test.go

@@ -112,9 +112,11 @@ func NewServer(config *Config) (*Server, error) {
 		buckets:  make(map[string]*bucket),
 		config:   config,
 	}
-	go func() { _ = http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		srv.serveHTTP(w, req)
-	})) }()
+	go func() {
+		_ = http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+			srv.serveHTTP(w, req)
+		}))
+	}()
 	return srv, nil
 }
 
@@ -527,14 +529,13 @@ func (bucketResource) post(a *action) interface{} {
 // and dashes (-). You can use uppercase letters for buckets only in the
 // US Standard region.
 //
-// Must start with a number or letter
+// # Must start with a number or letter
 //
-// Must be between 3 and 255 characters long
+// # Must be between 3 and 255 characters long
 //
 // There's one extra rule (Must not be formatted as an IP address (e.g., 192.168.5.4)
 // but the real S3 server does not seem to check that rule, so we will not
 // check it either.
-//
 func validBucketName(name string) bool {
 	if len(name) < 3 || len(name) > 255 {
 		return false

system/api_lib.py

@@ -34,6 +34,7 @@ class APITest(BaseTest):
                 "linkMethod": "symlink"
             }
         },
+        "enableMetricsEndpoint": True,
         "enableSwaggerEndpoint": True
     }
 

system/docker-wrapper

@@ -15,4 +15,4 @@ else
 fi
 
 cd /work/src
-sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go $cmd
+sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go GOCACHE=/work/src/.go/cache $cmd

system/files/aptly-dual.pub

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQGiBFL7pY8RBAC5uHg/9AuGJ7EF7RYty89IDLeqvlPe710eDQpJ+itsOaA/5rr3
+IV1LMlqHpM2rkZkAPpARwjrga2ByJ1ww77Zq2uPqJIO2LZYWTLXic9Zity2OVu3Z
+XwtdsqagIMfT5dAgNmhe5lL7qgGUwYcFFa52s7U4qO0z2FfwHW1IQrnMpwCg5RQh
+Uqs5iUKdDtoeQjX5mWgQhjEEAI1zfXUvvcOrRsDlGNKYZigZiWC6J46jeR8Nnf9C
+WwhXS2fzQaJyDq9DorkvPZgWUAaLLCdfGETqLzDKajynhS1+OnfFQNzvkvEPRBSb
+C5k+GOF2E1E9rGXb31+1XZTcdIprp4/F3RNLLWNUwfgPLWJx9NzHTYqgBStecHkC
+ySZRA/9PNFAbeJZ27HNuzoGnAa0piZDLeAAHsM1V6cosMh7U1IZqjZcrMC9YXNxH
+2D90PvoBvpufCMRzL/fOVPT1JzQGYoKIX17Nmzvdq/a4YyLWRODjvWXd94bae2Xd
+Vy03DYhfp8VOVJW6HuAX9JN6MKXSNxaibgOPjU822Hxd1iCIQ7QtQXB0bHkgVGVz
+dGVyIChkb24ndCB1c2UgaXQpIDx0ZXN0QGFwdGx5LmluZm8+iGIEExECACIFAlL7
+pY8CGyMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJECHbuJwW2z5t2sQAoNn+
+0cADZa66HZNY2qJi44Oq4hjaAJsHzj9JKAHEpdix5N7b6QvaZQZYhrkBDQRS+6WP
+EAQA9BX+kbIM6VJYoyY9vUHXfAF4E2y2M7vl9knZ+jMPfMbI7dE3gRJQb3mngST5
+7eZWawo1DNE6h3LbHsB4mpro9XLUXUMBgXRsOq4D5E0ygvDZ/tJhy0AwFiTOXKEs
+/erzmbF7j/TWh4LVHXFI9DrnN0+EeF/mQC/wzX7WGCKe70cAAwUEAMr7959zUYNp
+E3v4IquIJpD22bT/FiyQjFG8yGy36c+7mOP3VWi0lz5yFqqeR9NDFuLDSwOEi0nB
+zXNmimLy+hIwMaHjbQLjLODmy/T9wKCgeAmK1ygT6YBGJJflThZ05M80T5hBtRA9
+z2eoTn0wbi6MLmD/rbEt+lUPfSA4V0t2iEkEGBECAAkFAlL7pY8CGwwACgkQIdu4
+nBbbPm05hgCgvYatZXRbEdZ91jJCQi1KI7lJ5Y8AnjvrHU0g84mE45QZFegZzzQo
+9relmDMEZ3YCRhYJKwYBBAHaRw8BAQdAYDU0VSBcurX+uqAeR/w/XOLSZcghvOqz
+Y8yWdcj3HUy0L0FwdGx5IFNlY29uZGFyeSBTaWduaW5nIEtleSA8YXB0bHlAZXhh
+bXBsZS5jb20+iJYEExYKAD4WIQSu4W3wGDVPZ/5fXHK79OGUNOkeTgUCZ3YCRgIb
+AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRC79OGUNOkeTid/AP9A
+kIMn2qI5TqZgzrnPt7SN16VvpMppPb2H0m0P6knQKQD8DHcLcrqAl2cjcEuntv75
+gOnEvmPDAO6S1rc8UgcWdQQ=
+=XPoo
+-----END PGP PUBLIC KEY BLOCK-----

system/files/aptly3.sec

@@ -0,0 +1,11 @@
+-----BEGIN PGP PRIVATE KEY BLOCK-----
+
+lFgEZ3YCRhYJKwYBBAHaRw8BAQdAYDU0VSBcurX+uqAeR/w/XOLSZcghvOqzY8yW
+dcj3HUwAAP9lsZgE1YQfaS9xfVOSi3f91lbq13+U9FPdwxfiET0+bBFrtC9BcHRs
+eSBTZWNvbmRhcnkgU2lnbmluZyBLZXkgPGFwdGx5QGV4YW1wbGUuY29tPoiWBBMW
+CgA+FiEEruFt8Bg1T2f+X1xyu/ThlDTpHk4FAmd2AkYCGwMFCQPCZwAFCwkIBwIG
+FQoJCAsCBBYCAwECHgECF4AACgkQu/ThlDTpHk4nfwD/QJCDJ9qiOU6mYM65z7e0
+jdelb6TKaT29h9JtD+pJ0CkA/Ax3C3K6gJdnI3BLp7b++YDpxL5jwwDukta3PFIH
+FnUE
+=IXTY
+-----END PGP PRIVATE KEY BLOCK-----

system/files/corruptdb.go

@@ -20,7 +20,7 @@ func main() {
 	if err != nil {
 		log.Fatalf("Error opening DB %q: %s", dbPath, err)
 	}
-	defer db.Close()
+	defer func() { _ = db.Close() }()
 
 	keys := db.KeysByPrefix([]byte(prefix))
 	if len(keys) == 0 {

system/lib.py

@@ -272,6 +272,9 @@ class BaseTest(object):
             self.run_cmd([
                 self.gpgFinder.gpg2, "--import",
                 os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files") + "/aptly.sec"], expected_code=None)
+            self.run_cmd([
+                self.gpgFinder.gpg2, "--import",
+                os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files") + "/aptly3.sec"], expected_code=None)
 
         if self.fixtureGpg:
             self.run_cmd([self.gpgFinder.gpg, "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] +
@@ -310,7 +313,9 @@ class BaseTest(object):
 
         if command[0] == "aptly":
             aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test"
-            command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]]
+            command = [str(aptly_testing_bin), *command[1:]]
+            if self.coverage_dir is not None:
+                command.insert(1, f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out")
 
         if self.faketime:
             command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command
@@ -337,7 +342,7 @@ class BaseTest(object):
         if is_aptly_command:
             # remove the last two rows as go tests always print PASS/FAIL and coverage in those
             # two lines. This would otherwise fail the tests as they would not match gold
-            matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output)
+            matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*(?:\ncoverage: .*|$)", raw_output)
             if not matches:
                 raise Exception("no matches found in command output '%s'" % raw_output)
 
@@ -517,7 +522,7 @@ class BaseTest(object):
         if gold != output:
             diff = "".join(difflib.unified_diff(
                 [l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")]))
-            raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n")
+            raise Exception(f"content doesn't match:\n{diff}\n\nOutput:\n{orig}\n")
 
     check = check_output
 

system/run.py

@@ -36,7 +36,7 @@ def natural_key(string_):
     return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)]
 
 
-def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None):
+def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None, coverage_skip=False):
     """
     Run system test.
     """
@@ -47,7 +47,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non
     fails = []
     numTests = numFailed = numSkipped = 0
     lastBase = None
-    if not coverage_dir:
+    if not coverage_dir and not coverage_skip:
         coverage_dir = mkdtemp(suffix="aptly-coverage")
 
     failed = False
@@ -213,6 +213,7 @@ if __name__ == "__main__":
     include_long_tests = False
     capture_results = False
     coverage_dir = None
+    coverage_skip = False
     tests = None
     args = sys.argv[1:]
 
@@ -224,6 +225,8 @@ if __name__ == "__main__":
         elif args[0] == "--coverage-dir":
             coverage_dir = args[1]
             args = args[1:]
+        elif args[0] == "--coverage-skip":
+            coverage_skip = True
 
         args = args[1:]
 
@@ -236,4 +239,4 @@ if __name__ == "__main__":
         else:
             filters.append(arg)
 
-    run(include_long_tests, capture_results, tests, filters, coverage_dir)
+    run(include_long_tests, capture_results, tests, filters, coverage_dir, coverage_skip)

system/t02_config/ConfigShowTest_gold

@@ -12,6 +12,7 @@
     "dependencyVerboseResolve": false,
     "ppaDistributorID": "ubuntu",
     "ppaCodename": "",
+    "ppaBaseURL": "http://ppa.launchpad.net",
     "serveInAPIMode": true,
     "enableMetricsEndpoint": true,
     "enableSwaggerEndpoint": false,
@@ -29,6 +30,7 @@
     "gpgProvider": "gpg",
     "gpgDisableSign": false,
     "gpgDisableVerify": false,
+    "gpgKeys": [],
     "skipContentsPublishing": false,
     "skipBz2Publishing": false,
     "FileSystemPublishEndpoints": {},

system/t02_config/ConfigShowYAMLTest_gold

@@ -11,6 +11,7 @@ dep_follow_source: false
 dep_verboseresolve: false
 ppa_distributor_id: ubuntu
 ppa_codename: ""
+ppa_baseurl: http://ppa.launchpad.net
 serve_in_api_mode: true
 enable_metrics_endpoint: true
 enable_swagger_endpoint: false
@@ -27,6 +28,7 @@ download_sourcepackages: false
 gpg_provider: gpg
 gpg_disable_sign: false
 gpg_disable_verify: false
+gpg_keys: []
 skip_contents_publishing: false
 skip_bz2_publishing: false
 filesystem_publish_endpoints: {}

system/t02_config/CreateConfigTest_gold

@@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu
 # Codename for short PPA url expansion
 ppa_codename: ""
 
+# PPA Base URL (default: launchpad)
+# # ppa_baseurl: http://ppa.launchpad.net
+
 
 # Aptly Server
 ###############

system/t04_mirror/CreateMirror18Test_gold

@@ -1,4 +1,4 @@
-Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
+Downloading: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
 gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC
 gpgv:                using RSA key 5BFCD481D86D5824470E469F9000B1C3A01F726C
 gpgv: Good signature from "Launchpad PPA for Anton Gladky"
@@ -6,5 +6,5 @@ gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC
 gpgv:                using RSA key 02219381E9161C78A46CB2BFA5279A973B1F56C0
 gpgv: Good signature from "Launchpad sim"
 
-Mirror [mirror18]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick successfully added.
+Mirror [mirror18]: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/ maverick successfully added.
 You can run 'aptly mirror update mirror18' to download repository contents.

system/t04_mirror/CreateMirror18Test_mirror_show

@@ -1,5 +1,5 @@
 Name: mirror18
-Archive Root URL: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/
+Archive Root URL: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/
 Distribution: maverick
 Components: main
 Architectures: amd64, armel, i386, powerpc

system/t04_mirror/create.py

@@ -221,6 +221,7 @@ class CreateMirror18Test(BaseTest):
         "max-tries": 1,
         "ppaDistributorID": "ubuntu",
         "ppaCodename": "maverick",
+        "ppaBaseURL": "http://repo.aptly.info/system-tests/ppa",
     }
 
     fixtureCmds = [

system/t06_publish/PublishRepo36Test_gold

@@ -0,0 +1,12 @@
+Loading packages...
+Generating metadata files and linking package files...
+Finalizing metadata files...
+
+Local repo local-repo has been successfully published.
+Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing.
+Now you can add following line to apt sources:
+  deb http://your-server/ maverick main
+  deb-src http://your-server/ maverick main
+Don't forget to add your GPG key to apt with apt-key.
+
+You can also use `aptly serve` to publish your repositories over HTTP quickly.

system/t06_publish/PublishRepo36Test_release

@@ -0,0 +1,12 @@
+Origin: . maverick
+Label: . maverick
+Suite: maverick
+Codename: maverick
+Date: Fri, 13 Feb 2009 23:31:30 UTC
+Architectures: i386
+Components: main
+Description: Generated by aptly
+MD5Sum:
+SHA1:
+SHA256:
+SHA512:

system/t06_publish/repo.py

@@ -9,6 +9,10 @@ def strip_processor(output):
     return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:')])
 
 
+def strip_processor_keep_date(output):
+    return "\n".join([l for l in output.split("\n") if not l.startswith(' ')])
+
+
 class PublishRepo1Test(BaseTest):
     """
     publish repo: default
@@ -951,3 +955,34 @@ class PublishRepo34Test(BaseTest):
 
         if 'main/dep11/README' not in pathsSeen:
             raise Exception("README file not included in release file")
+
+
+class PublishRepo36Test(BaseTest):
+    """
+    publish repo: SOURCE_DATE_EPOCH produces byte-identical output
+    """
+    fixtureCmds = [
+        "aptly repo create local-repo",
+        "aptly repo add local-repo ${files}",
+    ]
+    runCmd = "aptly publish repo -skip-signing -distribution=maverick local-repo"
+    gold_processor = BaseTest.expand_environ
+    environmentOverride = {"SOURCE_DATE_EPOCH": "1234567890"}
+
+    def check(self):
+        super(PublishRepo36Test, self).check()
+
+        # verify Release file includes the expected date from SOURCE_DATE_EPOCH
+        self.check_file_contents(
+            'public/dists/maverick/Release', 'release', match_prepare=strip_processor_keep_date)
+
+        # save Release file from first publish
+        first_release = self.read_file('public/dists/maverick/Release')
+
+        # drop and republish with same SOURCE_DATE_EPOCH
+        self.run_cmd("aptly publish drop maverick")
+        self.run_cmd("aptly publish repo -skip-signing -distribution=maverick local-repo")
+
+        # verify byte-identical output
+        second_release = self.read_file('public/dists/maverick/Release')
+        self.check_equal(first_release, second_release)

system/t12_api/PublishAPITestDualSignature_Release.gpg

@@ -0,0 +1,14 @@
+gpg: Signature made Mon Jan 26 10:18:32 2026 UTC
+gpg:                using DSA key C5ACD2179B5231DFE842EE6121DBB89C16DB3E6D
+gpg: checking the trustdb
+gpg: no ultimately trusted keys found
+gpg: Good signature from "Aptly Tester (don't use it) <test@aptly.info>" [unknown]
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: C5AC D217 9B52 31DF E842  EE61 21DB B89C 16DB 3E6D
+gpg: Signature made Mon Jan 26 10:18:32 2026 UTC
+gpg:                using EDDSA key AEE16DF018354F67FE5F5C72BBF4E19434E91E4E
+gpg: Good signature from "Aptly Secondary Signing Key <aptly@example.com>" [unknown]
+gpg: WARNING: This key is not certified with a trusted signature!
+gpg:          There is no indication that the signature belongs to the owner.
+Primary key fingerprint: AEE1 6DF0 1835 4F67 FE5F  5C72 BBF4 E194 34E9 1E4E

system/t12_api/docs.py

@@ -1,17 +0,0 @@
-from api_lib import APITest
-
-
-class TaskAPITestSwaggerDocs(APITest):
-    """
-    GET /docs
-    """
-
-    def check(self):
-        resp = self.get("/docs/doc.json")
-        self.check_equal(resp.status_code, 200)
-
-        resp = self.get("/docs/", allow_redirects=False)
-        self.check_equal(resp.status_code, 301)
-
-        resp = self.get("/docs/index.html")
-        self.check_equal(resp.status_code, 200)

system/t12_api/mirrors.py

@@ -56,7 +56,6 @@ class MirrorsAPITestCreateUpdate(APITest):
         resp = self.get("/api/mirrors/" + mirror_name + "/packages")
         self.check_equal(resp.status_code, 404)
 
-        mirror_desc["Name"] = self.random_name()
         resp = self.put_task("/api/mirrors/" + mirror_name, json=mirror_desc)
         self.check_task(resp)
         _id = resp.json()['ID']

system/t12_api/publish.py

@@ -1,6 +1,7 @@
 import inspect
 import os
 import threading
+import re
 
 from api_lib import TASK_SUCCEEDED, APITest
 
@@ -221,8 +222,6 @@ class PublishSnapshotAPITest(APITest):
                 "Distribution": "squeeze",
                 "NotAutomatic": "yes",
                 "ButAutomaticUpgrades": "yes",
-                "Origin": "earth",
-                "Label": "fun",
             }
         )
         self.check_task(task)
@@ -237,8 +236,8 @@ class PublishSnapshotAPITest(APITest):
             'Architectures': ['i386'],
             'Codename': '',
             'Distribution': 'squeeze',
-            'Label': 'fun',
-            'Origin': 'earth',
+            'Label': '',
+            'Origin': '',
             'MultiDist': False,
             'NotAutomatic': 'yes',
             'ButAutomaticUpgrades': 'yes',
@@ -444,6 +443,156 @@ class PublishUpdateAPIMultiDist(APITest):
         self.check_not_exists("public/" + prefix + "dists/")
 
 
+class PublishUpdateAPIMultiDistToggle(APITest):
+    """
+    POST /publish/:prefix with MultiDist=false, then PUT to enable MultiDist=true
+    """
+    fixtureGpg = True
+
+    def check(self):
+        repo_name = self.random_name()
+        self.check_equal(self.post(
+            "/api/repos", json={"Name": repo_name, "DefaultDistribution": "bookworm"}).status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(self.upload("/api/files/" + d,
+                                     "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc",
+                                     "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
+                                     "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
+
+        task = self.post_task("/api/repos/" + repo_name + "/file/" + d)
+        self.check_task(task)
+
+        # Publish with MultiDist=false (default)
+        prefix = self.random_name()
+        task = self.post_task(
+            "/api/publish/" + prefix,
+            json={
+                "Architectures": ["i386", "source"],
+                "SourceKind": "local",
+                "Sources": [{"Name": repo_name}],
+                "Signing": DefaultSigningOptions,
+                "MultiDist": False,
+            }
+        )
+        self.check_task(task)
+
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'bookworm',
+            'Label': '',
+            'Origin': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Path': prefix + '/' + 'bookworm',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'MultiDist': False,
+            'SourceKind': 'local',
+            'Sources': [{'Component': 'main', 'Name': repo_name}],
+            'Storage': '',
+            'Suite': ''}
+
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        # With MultiDist=false packages are stored under pool/main/...
+        self.check_exists("public/" + prefix + "/dists/bookworm/Release")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/binary-i386/Packages")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/source/Sources")
+        self.check_exists(
+            "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists(
+            "public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+        # MultiDist-style per-distribution pool must not exist yet
+        self.check_not_exists(
+            "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+
+        # Now update the published repo enabling MultiDist=true
+        task = self.put_task(
+            "/api/publish/" + prefix + "/bookworm",
+            json={
+                "MultiDist": True,
+                "Signing": DefaultSigningOptions,
+            }
+        )
+        self.check_task(task)
+
+        repo_expected_multidist = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'bookworm',
+            'Label': '',
+            'Origin': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Path': prefix + '/' + 'bookworm',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'MultiDist': True,
+            'SourceKind': 'local',
+            'Sources': [{'Component': 'main', 'Name': repo_name}],
+            'Storage': '',
+            'Suite': ''}
+
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected_multidist, all_repos.json())
+
+        # After enabling MultiDist, packages are stored under pool/<distribution>/main/...
+        self.check_exists("public/" + prefix + "/dists/bookworm/Release")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/binary-i386/Packages")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/source/Sources")
+        self.check_exists(
+            "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists(
+            "public/" + prefix + "/pool/bookworm/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+        # Flat pool must not exist while MultiDist is on
+        self.check_not_exists(
+            "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+
+        # Switch MultiDist back to false
+        task = self.put_task(
+            "/api/publish/" + prefix + "/bookworm",
+            json={
+                "MultiDist": False,
+                "Signing": DefaultSigningOptions,
+            }
+        )
+        self.check_task(task)
+
+        repo_expected["MultiDist"] = False
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        # Packages are back under the flat pool/main/...
+        self.check_exists("public/" + prefix + "/dists/bookworm/Release")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/binary-i386/Packages")
+        self.check_exists("public/" + prefix +
+                          "/dists/bookworm/main/source/Sources")
+        self.check_exists(
+            "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists(
+            "public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+        # Per-distribution pool must be gone
+        self.check_not_exists(
+            "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+
+        task = self.delete_task("/api/publish/" + prefix + "/bookworm")
+        self.check_task(task)
+        self.check_not_exists("public/" + prefix + "dists/")
+
+
 class PublishConcurrentUpdateAPITestRepo(APITest):
     """
     PUT /publish/:prefix/:distribution (local repos), DELETE /publish/:prefix/:distribution
@@ -759,6 +908,219 @@ class PublishSwitchAPITestRepo(APITest):
         self.check_not_exists("public/" + prefix + "dists/")
 
 
+class PublishSwitchAPITestMirror(APITest):
+    """
+    PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution
+    """
+    fixtureGpg = True
+
+    def check(self):
+        mirror_name = self.random_name()
+        mirror_desc = {'Name': mirror_name,
+                       'ArchiveURL': 'http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/',
+                       'Distribution': 'wheezy',
+                       'Keyrings': ["aptlytest.gpg"],
+                       'Architectures': ["amd64"],
+                       'Components': ['main']}
+        mirror_desc['IgnoreSignatures'] = True
+
+        # Create Mirror
+        resp = self.post("/api/mirrors", json=mirror_desc)
+        self.check_equal(resp.status_code, 201)
+
+        # Get Mirror
+        resp = self.get("/api/mirrors/" + mirror_name + "/packages")
+        self.check_equal(resp.status_code, 404)
+
+        # Update Mirror
+        resp = self.put_task("/api/mirrors/" + mirror_name, json=mirror_desc)
+        self.check_task(resp)
+
+        # Snapshot Mirror
+        snapshot1_name = self.random_name()
+        task = self.post_task("/api/mirrors/" + mirror_name + '/snapshots', json={'Name': snapshot1_name})
+        self.check_task(task)
+
+        # Publish Snapshot
+        prefix = self.random_name()
+        task = self.post_task(
+            "/api/publish/" + prefix,
+            json={
+                "Architectures": ["i386", "source"],
+                "SourceKind": "snapshot",
+                "Sources": [{"Name": snapshot1_name}],
+                "Signing": DefaultSigningOptions,
+            })
+        self.check_task(task)
+
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'wheezy',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': 'packagecloud.io/varnishcache/varnish30',
+            'Path': prefix + '/' + 'wheezy',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'MultiDist': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot1_name}],
+            'Storage': '',
+            'Suite': ''}
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        # Snapshot Mirror 2
+        snapshot2_name = self.random_name()
+        task = self.post_task("/api/mirrors/" + mirror_name + '/snapshots', json={'Name': snapshot2_name})
+        self.check_task(task)
+
+        task = self.put_task(
+            "/api/publish/" + prefix + "/wheezy",
+            json={
+                "Snapshots": [{"Component": "main", "Name": snapshot2_name}],
+                "Signing": DefaultSigningOptions,
+                "SkipContents": True,
+                "Version": "13.3",
+            })
+        self.check_task(task)
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'wheezy',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': 'packagecloud.io/varnishcache/varnish30',
+            'Path': prefix + '/' + 'wheezy',
+            'Prefix': prefix,
+            'SkipContents': True,
+            'MultiDist': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot2_name}],
+            'Storage': '',
+            'Suite': ''}
+
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        task = self.delete_task("/api/publish/" + prefix + "/wheezy")
+        self.check_task(task)
+        self.check_not_exists("public/" + prefix + "dists/")
+
+
+class PublishSwitchAPITestSnapshot(APITest):
+    """
+    publish snapshot of snapshot
+    """
+    fixtureGpg = True
+
+    def check(self):
+        repo_name = self.random_name()
+        self.check_equal(self.post(
+            "/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(
+            self.upload("/api/files/" + d,
+                        "pyspi_0.6.1-1.3.dsc",
+                        "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
+                        "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
+        task = self.post_task("/api/repos/" + repo_name + "/file/" + d)
+        self.check_task(task)
+
+        snapshot1_name = self.random_name()
+        task = self.post_task("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot1_name})
+        self.check_task(task)
+
+        prefix = self.random_name()
+        task = self.post_task(
+            "/api/publish/" + prefix,
+            json={
+                "Architectures": ["i386", "source"],
+                "SourceKind": "snapshot",
+                "Sources": [{"Name": snapshot1_name}],
+                "Signing": DefaultSigningOptions,
+            })
+        self.check_task(task)
+
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'wheezy',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': '',
+            'Path': prefix + '/' + 'wheezy',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'MultiDist': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot1_name}],
+            'Storage': '',
+            'Suite': ''}
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        self.check_not_exists(
+            "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix +
+                          "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        snapshot2_name = self.random_name()
+        task = self.post_task("/api/snapshots", json={"Name": snapshot2_name, 'SourceSnapshots': [snapshot1_name]})
+        self.check_task(task)
+
+        task = self.put_task(
+            "/api/publish/" + prefix + "/wheezy",
+            json={
+                "Snapshots": [{"Component": "main", "Name": snapshot2_name}],
+                "Signing": DefaultSigningOptions,
+                "SkipContents": True,
+                "Version": "13.3",
+            })
+        self.check_task(task)
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'wheezy',
+            'Label': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Origin': '',
+            'Path': prefix + '/' + 'wheezy',
+            'Prefix': prefix,
+            'SkipContents': True,
+            'MultiDist': False,
+            'SourceKind': 'snapshot',
+            'Sources': [{'Component': 'main', 'Name': snapshot2_name}],
+            'Storage': '',
+            'Suite': ''}
+
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        self.check_not_exists(
+            "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb")
+        self.check_exists("public/" + prefix +
+                          "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc")
+
+        task = self.delete_task("/api/publish/" + prefix + "/wheezy")
+        self.check_task(task)
+        self.check_not_exists("public/" + prefix + "dists/")
+
+
 class PublishSwitchAPISkipCleanupTestRepo(APITest):
     """
     PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution
@@ -1557,3 +1919,62 @@ class PublishUpdateSourcesAPITestRepo(APITest):
         all_repos = self.get("/api/publish")
         self.check_equal(all_repos.status_code, 200)
         self.check_in(repo_expected, all_repos.json())
+
+
+class PublishAPITestDualSignature(APITest):
+    """
+    POST /publish/:prefix (local repos), GET /publish
+    """
+    fixtureGpg = True
+
+    def check(self):
+        repo_name = self.random_name()
+        self.check_equal(self.post(
+            "/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201)
+
+        d = self.random_name()
+        self.check_equal(self.upload("/api/files/" + d,
+                                     "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc",
+                                     "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz",
+                                     "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200)
+
+        task = self.post_task("/api/repos/" + repo_name + "/file/" + d)
+        self.check_task(task)
+
+        # publishing under prefix, default distribution
+        prefix = self.random_name()
+        task = self.post_task(
+            "/api/publish/" + prefix,
+            json={
+                 "SourceKind": "local",
+                 "Sources": [{"Name": repo_name}],
+                 "Signing": {"GPGKey": "C5ACD2179B5231DFE842EE6121DBB89C16DB3E6D,AEE16DF018354F67FE5F5C72BBF4E19434E91E4E"},
+            }
+        )
+        self.check_task(task)
+        repo_expected = {
+            'AcquireByHash': False,
+            'Architectures': ['i386', 'source'],
+            'Codename': '',
+            'Distribution': 'wheezy',
+            'Label': '',
+            'Origin': '',
+            'NotAutomatic': '',
+            'ButAutomaticUpgrades': '',
+            'Path': prefix + '/' + 'wheezy',
+            'Prefix': prefix,
+            'SkipContents': False,
+            'MultiDist': False,
+            'SourceKind': 'local',
+            'Sources': [{'Component': 'main', 'Name': repo_name}],
+            'Storage': '',
+            'Suite': ''}
+
+        all_repos = self.get("/api/publish")
+        self.check_equal(all_repos.status_code, 200)
+        self.check_in(repo_expected, all_repos.json())
+
+        self.check_exists("public/" + prefix + "/dists/wheezy/Release")
+        path = os.path.join(os.environ["HOME"], self.aptlyDir, "public", prefix, "dists/wheezy")
+        self.check_cmd_output(f"gpg --verify {path}/Release.gpg {path}/Release", "Release.gpg",
+                              match_prepare=lambda s: re.sub(r'Signature made .*', '', s))

system/t12_api/repos.py

@@ -461,3 +461,34 @@ class ReposAPITestCopyPackage(APITest):
 
         self.check_equal(self.get(f"/api/repos/{repo2_name}/packages").json(),
                          ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378'])
+
+
+class ReposAPITestCreateEdit(APITest):
+    """
+    POST /api/repos,
+    """
+    def check(self):
+        repo_name = self.random_name() + ' with space'
+        repo_desc = {'Comment': 'fun repo',
+                     'DefaultComponent': 'contrib',
+                     'DefaultDistribution': 'bookworm',
+                     'Name': repo_name}
+
+        resp = self.post("/api/repos", json=repo_desc)
+        self.check_equal(resp.json(), repo_desc)
+        self.check_equal(resp.status_code, 201)
+
+        repo_desc = {'Comment': 'modified repo',
+                     'DefaultComponent': 'main',
+                     'DefaultDistribution': 'trixie',
+                     'Name': repo_name + '@renamed'}
+        resp = self.put(f"/api/repos/{repo_name}", json=repo_desc)
+        self.check_equal(resp.json(), repo_desc)
+        self.check_equal(resp.status_code, 200)
+
+        resp = self.get("/api/repos/" + repo_name + '@renamed')
+        self.check_equal(resp.json(), repo_desc)
+        self.check_equal(resp.status_code, 200)
+
+        resp = self.delete("/api/repos/" + repo_name + '@renamed')
+        self.check_equal(resp.status_code, 200)

system/testout.py

@@ -3,7 +3,7 @@ import tempfile
 
 class TestOut:
     def __init__(self):
-        self.tmp_file = tempfile.NamedTemporaryFile(delete=False)
+        self.tmp_file = tempfile.NamedTemporaryFile(delete=True)
         self.read_pos = 0
 
     def fileno(self):

task/list.go

@@ -44,45 +44,53 @@ func (list *List) consumer() {
 	for {
 		select {
 		case task := <-list.queue:
+			// Set task state to RUNNING before processing
 			list.Lock()
-			{
-				task.State = RUNNING
-			}
+			task.State = RUNNING
 			list.Unlock()
 
 			go func() {
 				retValue, err := task.process(aptly.Progress(task.output), task.detail)
 
+				// Update task completion state and cleanup with list lock held
 				list.Lock()
 				{
-					task.processReturnValue = retValue
-					task.err = err
 					if err != nil {
 						task.output.Printf("Task failed with error: %v", err)
 						task.State = FAILED
+						task.err = err
+						task.processReturnValue = retValue
 					} else {
 						task.output.Print("Task succeeded")
 						task.State = SUCCEEDED
+						task.err = nil
+						task.processReturnValue = retValue
 					}
 
-					list.usedResources.Free(task.resources)
+					list.usedResources.Free(task.Resources)
 
 					task.wgTask.Done()
 					list.wg.Done()
 
+					unlocked := false
 					for _, t := range list.tasks {
 						if t.State == IDLE {
 							// check resources
-							blockingTasks := list.usedResources.UsedBy(t.resources)
+							blockingTasks := list.usedResources.UsedBy(t.Resources)
 							if len(blockingTasks) == 0 {
-								list.usedResources.MarkInUse(task.resources, task)
+								list.usedResources.MarkInUse(t.Resources, t)
+								// unlock list since queueing may block
+								list.Unlock()
+								unlocked = true
 								list.queue <- t
 								break
 							}
 						}
 					}
+					if !unlocked {
+						list.Unlock()
+					}
 				}
-				list.Unlock()
 			}()
 
 		case <-list.queueDone:
@@ -99,13 +107,15 @@ func (list *List) Stop() {
 
 // GetTasks gets complete list of tasks
 func (list *List) GetTasks() []Task {
-	tasks := []Task{}
 	list.Lock()
+	defer list.Unlock()
+
+	tasks := []Task{}
 	for _, task := range list.tasks {
+		// Copy task while holding list lock
 		tasks = append(tasks, *task)
 	}
 
-	list.Unlock()
 	return tasks
 }
 
@@ -133,11 +143,11 @@ func (list *List) DeleteTaskByID(ID int) (Task, error) {
 // GetTaskByID returns task with given id
 func (list *List) GetTaskByID(ID int) (Task, error) {
 	list.Lock()
-	tasks := list.tasks
-	list.Unlock()
+	defer list.Unlock()
 
-	for _, task := range tasks {
+	for _, task := range list.tasks {
 		if task.ID == ID {
+			// Copy task while holding list lock
 			return *task, nil
 		}
 	}
@@ -174,20 +184,22 @@ func (list *List) GetTaskDetailByID(ID int) (interface{}, error) {
 
 // GetTaskReturnValueByID returns process return value of task with given id
 func (list *List) GetTaskReturnValueByID(ID int) (*ProcessReturnValue, error) {
-	task, err := list.GetTaskByID(ID)
+	list.Lock()
+	defer list.Unlock()
 
-	if err != nil {
-		return nil, err
+	for _, task := range list.tasks {
+		if task.ID == ID {
+			return task.processReturnValue, nil
+		}
 	}
 
-	return task.processReturnValue, nil
+	return nil, fmt.Errorf("could not find task with id %v", ID)
 }
 
 // RunTaskInBackground creates task and runs it in background. This will block until the necessary resources
 // become available.
 func (list *List) RunTaskInBackground(name string, resources []string, process Process) (Task, *ResourceConflictError) {
 	list.Lock()
-	defer list.Unlock()
 
 	list.idCounter++
 	wgTask := &sync.WaitGroup{}
@@ -199,20 +211,29 @@ func (list *List) RunTaskInBackground(name string, resources []string, process P
 	list.wg.Add(1)
 	task.wgTask.Add(1)
 
+	// Copy task while still holding the lock to avoid racing with consumer
+	// setting State=RUNNING after receiving from queue
+	taskCopy := *task
+
 	// add task to queue for processing if resources are available
 	// if not, task will be queued by the consumer once resources are available
 	tasks := list.usedResources.UsedBy(resources)
 	if len(tasks) == 0 {
-		list.usedResources.MarkInUse(task.resources, task)
+		list.usedResources.MarkInUse(task.Resources, task)
+		// queueing task might block if channel not ready, unlock list before queueing
+		list.Unlock()
 		list.queue <- task
+	} else {
+		list.Unlock()
 	}
 
-	return *task, nil
+	return taskCopy, nil
 }
 
 // Clear removes finished tasks from list
 func (list *List) Clear() {
 	list.Lock()
+	defer list.Unlock()
 
 	var tasks []*Task
 	for _, task := range list.tasks {
@@ -221,8 +242,6 @@ func (list *List) Clear() {
 		}
 	}
 	list.tasks = tasks
-
-	list.Unlock()
 }
 
 // Wait waits till all tasks are processed
@@ -245,11 +264,14 @@ func (list *List) WaitForTaskByID(ID int) (Task, error) {
 
 // GetTaskErrorByID returns the Task error for a given id
 func (list *List) GetTaskErrorByID(ID int) (error, error) {
-	task, err := list.GetTaskByID(ID)
+	list.Lock()
+	defer list.Unlock()
 
-	if err != nil {
-		return nil, err
+	for _, task := range list.tasks {
+		if task.ID == ID {
+			return task.err, nil
+		}
 	}
 
-	return task.err, nil
+	return nil, fmt.Errorf("could not find task with id %v", ID)
 }

task/list_test.go

@@ -50,5 +50,5 @@ func (s *ListSuite) TestList(c *check.C) {
 	c.Check(detail, check.Equals, "Details")
 	_, deleteErr := list.DeleteTaskByID(task.ID)
 	c.Check(deleteErr, check.IsNil)
-        list.Stop()
+	list.Stop()
 }

task/task.go

@@ -42,6 +42,7 @@ const (
 )
 
 // Task represents as task in a queue encapsulates process code
+// All fields are protected by List.Mutex - access task fields only while holding list.Lock()
 type Task struct {
 	output             *Output
 	detail             *Detail
@@ -51,7 +52,7 @@ type Task struct {
 	Name               string
 	ID                 int
 	State              State
-	resources          []string
+	Resources          []string
 	wgTask             *sync.WaitGroup
 }
 
@@ -64,7 +65,7 @@ func NewTask(process Process, name string, ID int, resources []string, wgTask *s
 		Name:      name,
 		ID:        ID,
 		State:     IDLE,
-		resources: resources,
+		Resources: resources,
 		wgTask:    wgTask,
 	}
 	return task

utils/config.go

@@ -31,6 +31,7 @@ type ConfigStructure struct { // nolint: maligned
 	// PPA
 	PpaDistributorID string `json:"ppaDistributorID"              yaml:"ppa_distributor_id"`
 	PpaCodename      string `json:"ppaCodename"                   yaml:"ppa_codename"`
+	PpaBaseURL       string `json:"ppaBaseURL"                    yaml:"ppa_baseurl"`
 
 	// Server
 	ServeInAPIMode        bool `json:"serveInAPIMode"                yaml:"serve_in_api_mode"`
@@ -49,9 +50,10 @@ type ConfigStructure struct { // nolint: maligned
 	DownloadSourcePackages bool   `json:"downloadSourcePackages"        yaml:"download_sourcepackages"`
 
 	// Signing
-	GpgProvider      string `json:"gpgProvider"                   yaml:"gpg_provider"`
-	GpgDisableSign   bool   `json:"gpgDisableSign"                yaml:"gpg_disable_sign"`
-	GpgDisableVerify bool   `json:"gpgDisableVerify"              yaml:"gpg_disable_verify"`
+	GpgProvider      string   `json:"gpgProvider"                   yaml:"gpg_provider"`
+	GpgDisableSign   bool     `json:"gpgDisableSign"                yaml:"gpg_disable_sign"`
+	GpgDisableVerify bool     `json:"gpgDisableVerify"              yaml:"gpg_disable_verify"`
+	GpgKeys          []string `json:"gpgKeys"                       yaml:"gpg_keys"`
 
 	// Publishing
 	SkipContentsPublishing bool `json:"skipContentsPublishing"        yaml:"skip_contents_publishing"`
@@ -226,6 +228,7 @@ var Config = ConfigStructure{
 	GpgProvider:            "gpg",
 	GpgDisableSign:         false,
 	GpgDisableVerify:       false,
+	GpgKeys:                []string{},
 	DownloadSourcePackages: false,
 	PackagePoolStorage: PackagePoolStorage{
 		Local: &LocalPoolStorage{Path: ""},
@@ -233,6 +236,7 @@ var Config = ConfigStructure{
 	SkipLegacyPool:         false,
 	PpaDistributorID:       "ubuntu",
 	PpaCodename:            "",
+	PpaBaseURL:             "http://ppa.launchpad.net",
 	FileSystemPublishRoots: map[string]FileSystemPublishRoot{},
 	S3PublishRoots:         map[string]S3PublishRoot{},
 	SwiftPublishRoots:      map[string]SwiftPublishRoot{},

utils/config_test.go

@@ -85,6 +85,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"  \"dependencyVerboseResolve\": false,\n" +
 		"  \"ppaDistributorID\": \"\",\n" +
 		"  \"ppaCodename\": \"\",\n" +
+		"  \"ppaBaseURL\": \"\",\n" +
 		"  \"serveInAPIMode\": false,\n" +
 		"  \"enableMetricsEndpoint\": false,\n" +
 		"  \"enableSwaggerEndpoint\": false,\n" +
@@ -102,6 +103,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) {
 		"  \"gpgProvider\": \"gpg\",\n" +
 		"  \"gpgDisableSign\": false,\n" +
 		"  \"gpgDisableVerify\": false,\n" +
+		"  \"gpgKeys\": null,\n" +
 		"  \"skipContentsPublishing\": false,\n" +
 		"  \"skipBz2Publishing\": false,\n" +
 		"  \"FileSystemPublishEndpoints\": {\n" +
@@ -251,6 +253,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) {
 		"dep_verboseresolve: false\n" +
 		"ppa_distributor_id: \"\"\n" +
 		"ppa_codename: \"\"\n" +
+		"ppa_baseurl: \"\"\n" +
 		"serve_in_api_mode: false\n" +
 		"enable_metrics_endpoint: false\n" +
 		"enable_swagger_endpoint: false\n" +
@@ -267,6 +270,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) {
 		"gpg_provider: \"\"\n" +
 		"gpg_disable_sign: false\n" +
 		"gpg_disable_verify: false\n" +
+		"gpg_keys: []\n" +
 		"skip_contents_publishing: false\n" +
 		"skip_bz2_publishing: false\n" +
 		"filesystem_publish_endpoints: {}\n" +
@@ -306,6 +310,7 @@ dep_follow_source: true
 dep_verboseresolve: true
 ppa_distributor_id: Ubuntu
 ppa_codename: code
+ppa_baseurl: http://ppa.launchpad.net
 serve_in_api_mode: true
 enable_metrics_endpoint: true
 enable_swagger_endpoint: true
@@ -322,6 +327,7 @@ download_sourcepackages: true
 gpg_provider: gpg
 gpg_disable_sign: true
 gpg_disable_verify: true
+gpg_keys: []
 skip_contents_publishing: true
 skip_bz2_publishing: true
 filesystem_publish_endpoints:

utils/utils_test.go

@@ -34,7 +34,7 @@ func (s *UtilsSuite) TestDirIsAccessibleNotExist(c *C) {
 func (s *UtilsSuite) TestDirIsAccessibleNotAccessible(c *C) {
 	accessible := DirIsAccessible(s.tempfile.Name())
 	if accessible == nil {
-            c.Fatalf("Test dir should not be accessible: %s", s.tempfile.Name())
-        }
+		c.Fatalf("Test dir should not be accessible: %s", s.tempfile.Name())
+	}
 	c.Check(accessible.Error(), Equals, fmt.Errorf("'%s' is inaccessible, check access rights", s.tempfile.Name()).Error())
 }