(not4debian) ci: run unit tests in docker

- run separate unit-test job
- build docker
- allow make docker-unit-tests in ci

.github/workflows/ci.yml

@@ -17,6 +17,29 @@ env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
+  unit-test:
+    name: "Unit Tests (Debian 13)"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+      - name: "Docker Image"
+        run: |
+          make docker-image
+      - name: "Unit Test"
+        run: |
+          make docker-unit-test
+      - name: "Upload Code Coverage"
+        uses: codecov/codecov-action@v2
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: unit.out
+
   test:
     name: "Test (Ubuntu 22.04)"
     runs-on: ubuntu-22.04
@@ -30,12 +53,12 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: "Install packages"
+      - name: "Install Test Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
 
-      - name: "Checkout repository"
+      - name: "Checkout Repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
@@ -45,7 +68,7 @@ jobs:
         run: |
           make flake8
 
-      - name: "Read go version from go.mod"
+      - name: "Read Go Version"
         run: |
           gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
           echo "Go Version: $gover"
@@ -63,18 +86,6 @@ jobs:
         with:
           directory: ${{ runner.temp }}
 
-      - name: "Run Unit Tests"
-        env:
-          RUN_LONG_TESTS: 'yes'
-          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
-          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
-          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        run: |
-          sudo mkdir -p /srv ; sudo chown runner /srv
-          COVERAGE_DIR=${{ runner.temp }} make test
-
       - name: "Run Benchmark"
         run: |
           COVERAGE_DIR=${{ runner.temp }} make bench
@@ -91,12 +102,13 @@ jobs:
           sudo mkdir -p /srv ; sudo chown runner /srv
           COVERAGE_DIR=${{ runner.temp }} make system-test
 
-      - name: "Merge code coverage"
+      - name: "Merge Code Coverage"
         run: |
-          go install github.com/wadey/gocovmerge@latest
-          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+          # go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
+          # ~/go/bin/gocovmerge coverage/*.out > coverage.txt
+          awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
 
-      - name: "Upload code coverage"
+      - name: "Upload Code Coverage"
         uses: codecov/codecov-action@v2
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -104,7 +116,8 @@ jobs:
 
   ci-debian-build:
     name: "Build"
-    needs: test
+    needs:
+      - test
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -139,7 +152,7 @@ jobs:
         APT_LISTCHANGES_FRONTEND: none
         DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: "Install packages"
+      - name: "Install Build Packages"
         run: |
           apt-get update
           apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
@@ -148,13 +161,13 @@ jobs:
                           gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
 
-      - name: "Checkout repository"
+      - name: "Checkout Repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: "Read go version from go.mod"
+      - name: "Read Go Version"
         run: |
           gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
           echo "Go Version: $gover"
@@ -207,9 +220,28 @@ jobs:
         run: |
           .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
 
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Upload CI Artifacts"
+        if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags')
+        uses: actions/upload-artifact@v4
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }}
+          path: build/
+          retention-days: 7
+
   ci-binary-build:
     name: "Build"
-    needs: test
+    needs:
+      - unit-test
+      - test
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -221,13 +253,13 @@ jobs:
           - goos: darwin
             goarch: arm
     steps:
-      - name: "Checkout repository"
+      - name: "Checkout Repository"
         uses: actions/checkout@v4
         with:
           # fetch the whole repo for `git describe` to work
           fetch-depth: 0
 
-      - name: "Read go version from go.mod"
+      - name: "Read Go Version"
         run: |
           echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
         id: goversion
@@ -267,6 +299,15 @@ jobs:
           path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
           compression-level: 0 # no compression
 
+      - name: "Upload CI Artifacts"
+        uses: actions/upload-artifact@v4
+        if: "!startsWith(github.event.ref, 'refs/tags')"
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+          retention-days: 7
+
   gh-release:
     name: "Github Release"
     runs-on: ubuntu-latest
@@ -274,12 +315,31 @@ jobs:
     needs: ci-binary-build
     if: startsWith(github.event.ref, 'refs/tags')
     steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
       - name: "Download Artifacts"
         uses: actions/download-artifact@v4
         with:
           path: out/
 
+      - name: "Create Release Notes"
+        run: |
+          echo -e "## Changes\n\n" > out/release-notes.md
+          dpkg-parsechangelog -S Changes | tail -n +4 >> out/release-notes.md
+
       - name: "Release"
         uses: softprops/action-gh-release@v2
         with:
+          name: "Aptly Release ${{ steps.releaseversion.outputs.VERSION }}"
           files: "out/**/aptly_*.zip"
+          body_path: "out/release-notes.md"

.github/workflows/golangci-lint.yml

@@ -44,7 +44,7 @@ jobs:
           # Require: The version of golangci-lint to use.
           # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
           # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
-          version: v1.54.1
+          version: v1.64.5
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

.github/workflows/scripts/upload-artifacts.sh

@@ -67,11 +67,14 @@ cleanup() {
 }
 trap cleanup EXIT
 
+sleeptime=5
+retries=60
 wait_task()
 {
     _id=$1
     _success=0
-    for t in `seq 180`
+    sleep $sleeptime
+    for t in `seq $retries`
     do
         jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_id`
         _state=`echo $jsonret | jq .State`
@@ -84,7 +87,7 @@ wait_task()
             echo Error: task failed
             return 1
         fi
-        sleep 1
+        sleep $sleeptime
     done
     if [ "$_success" -ne 1 ]; then
         echo Error: task timeout

.golangci.yml

@@ -1,16 +1,11 @@
-run:
-  tests: false
-
-
+version: "2"
 linters:
-  disable-all: true
-  enable:
-   - goconst
-   - gofmt
-   - goimports
-   - govet
-   - ineffassign
-   - misspell
-   - revive
-   - staticcheck
-   - vetshadow
+  settings:
+    staticcheck:
+      checks:
+      - "all"
+      - "-QF1004"  # could use strings.ReplaceAll instead
+      - "-QF1012"  # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))
+      - "-QF1003"  # could use tagged switch
+      - "-ST1000"  # at least one file in a package should have a package comment
+      - "-QF1001"  # could apply De Morgan's law

AUTHORS

@@ -68,3 +68,11 @@ List of contributors, in chronological order:
 * Blake Kostner (https://github.com/btkostner)
 * Leigh London (https://github.com/leighlondon)
 * Gordian Schoenherr (https://github.com/schoenherrg)
+* Silke Hofstra (https://github.com/silkeh)
+* Itay Porezky (https://github.com/itayporezky)
+* JupiterRider (https://github.com/JupiterRider)
+* Tobias Assarsson (https://github.com/daedaluz)
+* Yaksh Bariya (https://github.com/thunder-coding)
+* Brian Witt (https://github.com/bwitt)
+* Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu
 ### List of Repositories
 
 * [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
-* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
 

Makefile

@@ -2,13 +2,32 @@ GOPATH=$(shell go env GOPATH)
 VERSION=$(shell make -s version)
 PYTHON?=python3
 BINPATH?=$(GOPATH)/bin
-GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+GOLANGCI_LINT_VERSION=v2.0.2  # version supporting go 1.24
 COVERAGE_DIR?=$(shell mktemp -d)
 GOOS=$(shell go env GOHOSTOS)
 GOARCH=$(shell go env GOHOSTARCH)
 
-# Uncomment to update system test gold files
-# CAPTURE := "--capture"
+export PODMAN_USERNS = keep-id
+DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
+
+# Setting TZ for certificates
+export TZ=UTC
+# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
+export TEST_FAKETIME := 2025-01-02 03:04:05
+
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
+else
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
+endif
+
+# export CAPUTRE=1 for regenrating test gold files
+ifeq ($(CAPTURE),1)
+CAPTURE_ARG := --capture
+endif
 
 help:  ## Print this help
 	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -50,15 +69,15 @@ swagger-install:
 	echo "// @version $(VERSION)" >> docs/swagger.conf
 
 azurite-start:
-	azurite -l /tmp/aptly-azurite & \
+	azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
 	echo $$! > ~/.azurite.pid
 
 azurite-stop:
 	@kill `cat ~/.azurite.pid`
 
-swagger: swagger-install
+swagger: #swagger-install
 	# Generate swagger docs
-	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+	#@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
 
 etcd-install:
 	# Install etcd
@@ -69,9 +88,9 @@ flake8:  ## run flake8 on system test python files
 
 lint: prepare
 	# Install golangci-lint
-	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 	# Running lint
-	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+	@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0
 
 
 build: prepare swagger  ## Build aptly
@@ -84,11 +103,11 @@ install:
 	# go install -v
 	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-test: prepare swagger etcd-install  ## Run unit tests
+test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify which tests to run)
 	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
 	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
 	@echo "\e[33m\e[1mRunning go test ...\e[0m"
-	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
 	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
 	@pid=`cat /tmp/etcd.pid`; kill $$pid
 	@rm -f /tmp/aptly-etcd-data/etcd.log
@@ -96,13 +115,13 @@ test: prepare swagger etcd-install  ## Run unit tests
 
 system-test: prepare swagger etcd-install  ## Run system tests
 	# build coverage binary
-	go test -v -coverpkg="./..." -c -tags testruncli
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
 	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
 	# Run system tests
-	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
 
 bench:
 	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -112,6 +131,7 @@ serve: prepare swagger-install  ## Run development server (auto recompiling)
 	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf ~/.aptly.conf
 	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
 	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
 dpkg: prepare swagger  ## Build debian packages
@@ -140,7 +160,7 @@ dpkg: prepare swagger  ## Build debian packages
 	mkdir -p build && mv ../*.deb build/ ; \
 	cd build && ls -l *.deb
 
-binaries: prepare swagger  ## Build binary releases (FreeBSD, MacOS, Linux tar)
+binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
 	# build aptly
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
 	# install
@@ -162,44 +182,49 @@ binaries: prepare swagger  ## Build binary releases (FreeBSD, MacOS, Linux tar)
 docker-image:  ## Build aptly-dev docker image
 	@docker build -f system/Dockerfile . -t aptly-dev
 
+docker-image-no-cache:  ## Build aptly-dev docker image (no cache)
+	@docker build --no-cache -f system/Dockerfile . -t aptly-dev
+
 docker-build:  ## Build aptly in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
 
 docker-shell:  ## Run aptly and other commands in docker container
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+	@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
 
 docker-deb:  ## Build debian packages in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
 
-docker-unit-test:  ## Run unit tests in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
+	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
-		test \
+		test TEST=$(TEST) \
 		azurite-stop
 
 docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
-		system-test TEST=$(TEST) \
+		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
+		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
 		azurite-stop
 
 docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+	@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
 
 docker-lint:  ## Run golangci-lint in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
 
-docker-binaries:  ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
 
 docker-man:  ## Create man page in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
@@ -216,4 +241,4 @@ clean:  ## remove local build and module cache
 	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
 	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -1,7 +1,3 @@
-=====
-aptly
-=====
-
 .. image:: https://github.com/aptly-dev/aptly/actions/workflows/ci.yml/badge.svg
     :target: https://github.com/aptly-dev/aptly/actions
 
@@ -14,15 +10,18 @@ aptly
 .. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
     :target: https://goreportcard.com/report/aptly-dev/aptly
 
+aptly
+=====
+
 Aptly is a swiss army knife for Debian repository management.
 
 .. image:: http://www.aptly.info/img/aptly_logo.png
     :target: http://www.aptly.info/
 
 Documentation is available at `http://www.aptly.info/ <http://www.aptly.info/>`_. For support please use
-mailing list `aptly-discuss <https://groups.google.com/forum/#!forum/aptly-discuss>`_.
+open `issues <https://github.com/aptly-dev/aptly/issues>`_ or `discussions <https://github.com/aptly-dev/aptly/discussions>`_.
 
-Aptly features: ("+" means planned features)
+Aptly features:
 
 * make mirrors of remote Debian/Ubuntu repositories, limiting by components/architectures
 * take snapshots of mirrors at any point in time, fixing state of repository at some moment of time
@@ -32,48 +31,50 @@ Aptly features: ("+" means planned features)
 * filter repository by search query, pulling dependencies when required
 * publish self-made packages as Debian repositories
 * REST API for remote access
-* mirror repositories "as-is" (without resigning with user's key) (+)
-* support for yum repositories (+)
 
-Current limitations:
+Any contributions are welcome! Please see `CONTRIBUTING.md <CONTRIBUTING.md>`_.
 
-* translations are not supported yet
+Installation
+=============
 
-Install Stable Version
------------------------
+Aptly can be installed on several operating systems.
 
-To install aptly on Debian/Ubuntu, add new repository to ``/etc/apt/sources.list``::
+Debian / Ubuntu
+----------------
 
-    deb http://repo.aptly.info/ squeeze main
+Aptly is provided in the following debian packages:
 
-And import key that is used to sign the release::
+* **aptly**: Includes the main Aptly binary, man pages, and shell completions
+* **aptly-api**: A systemd service for the REST API, using the global /etc/aptly.conf
+* **aptly-dbg**: Debug symbols for troubleshooting
 
-    $ apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EE727D4449467F0E
+The packages can be installed on official `Debian <https://packages.debian.org/search?keywords=aptly>`_ and `Ubuntu <https://packages.ubuntu.com/search?keywords=aptly>`_ distributions.
 
-After that you can install aptly as any other software package::
+Upstream Debian Packages
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    $ apt-get update
-    $ apt-get install aptly
+If a newer version (not available in Debian/Ubuntu) of aptly is required, upstream debian packages (built from git tags) can be installed as follows:
 
-Don't worry about squeeze part in repo name: aptly package should work on Debian squeeze+,
-Ubuntu 10.0+. Package contains aptly binary, man page and bash completion.
+Install the following APT key (as root)::
 
-Other Binaries
-~~~~~~~~~~~~~~~~~
+    wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
 
-Binary executables (depends almost only on libc) are available for download from `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_.
+Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
 
-Install CI Version
---------------------
+    deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
 
-More recent versions are available as CI builds (development, might be unstable).
+Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
-Debian GNU/Linux
-~~~~~~~~~~~~~~~~~
+Install aptly packages::
 
-Install the following APT key::
+    apt-get update
+    apt-get install aptly
+    apt-get install aptly-api  # REST API systemd service
 
-    sudo wget -O /etc/apt/keyrings/aptly.asc https://www.aptly.info/pubkey.txt
+CI Builds
+~~~~~~~~~~
+
+For testing new features or bugfixes, recent builds are available as CI builds (built from master, may be unstable!) and can be installed as follows:
 
 Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
 
@@ -81,13 +82,19 @@ Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
 
 Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
 
-Contributing
-------------
+Note: same gpg key is used as for the Upstream Debian Packages.
 
-Please follow detailed documentation in `CONTRIBUTING.md <CONTRIBUTING.md>`_.
+Other Operating Systems
+------------------------
+
+Binary executables (depends almost only on libc) are available on `GitHub Releases <https://github.com/aptly-dev/aptly/releases>`_ for:
+
+- macOS / darwin (amd64, arm64)
+- FreeBSD (amd64, arm64, 386, arm)
+- Generic Linux (amd64, arm64, 386, arm)
 
 Integrations
-------------
+=============
 
 Vagrant:
 
@@ -124,3 +131,7 @@ GUI for aptly API:
 Scala sbt:
 
 -   `sbt aptly plugin <https://github.com/amalakar/sbt-aptly>`_ by Arup Malakar
+
+Molior:
+
+-   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth

Releasing.md

@@ -0,0 +1,17 @@
+# Creating a Release
+
+- create branch release/1.x.y
+- update debian/changelog
+- create PR, merge when approved
+- on updated master, create release:
+```
+version=$(dpkg-parsechangelog -S Version)
+echo Releasing prod version $version
+git tag -a v$version -m 'aptly: release $version'
+git push origin v$version master
+```
+- run swagger locally (`make docker-serve`)
+- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
+- add new version to select tag in content/doc/api/swagger.md line 48
+- push commit to master
+- create release announcement on https://github.com/aptly-dev/aptly/discussions

api/api.go

@@ -41,7 +41,10 @@ type aptlyVersion struct {
 // @Success 200 {object} aptlyVersion
 // @Router /api/version [get]
 func apiVersion(c *gin.Context) {
-	c.JSON(200, gin.H{"Version": aptly.Version})
+	version := aptlyVersion{
+		Version: aptly.Version,
+	}
+	c.JSON(200, version)
 }
 
 type aptlyStatus struct {
@@ -67,7 +70,8 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) {
 			return
 		}
 
-		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+		status := aptlyStatus{Status: "Aptly is ready"}
+		c.JSON(200, status)
 	}
 }
 
@@ -165,7 +169,7 @@ func runTaskInBackground(name string, resources []string, proc task.Process) (ta
 			return nil, err
 		}
 
-		defer releaseDatabaseConnection()
+		defer func() { _ = releaseDatabaseConnection() }()
 		return proc(out, detail)
 	})
 }
@@ -174,18 +178,18 @@ func truthy(value interface{}) bool {
 	if value == nil {
 		return false
 	}
-	switch value.(type) {
+	switch v := value.(type) {
 	case string:
-		switch strings.ToLower(value.(string)) {
+		switch strings.ToLower(v) {
 		case "n", "no", "f", "false", "0", "off":
 			return false
 		default:
 			return true
 		}
 	case int:
-		return !(value.(int) == 0)
+		return v != 0
 	case bool:
-		return value.(bool)
+		return v
 	}
 	return true
 }
@@ -210,11 +214,11 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 		}
 
 		// wait for task to finish
-		context.TaskList().WaitForTaskByID(task.ID)
+		_, _ = context.TaskList().WaitForTaskByID(task.ID)
 
 		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
 		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
-		context.TaskList().DeleteTaskByID(task.ID)
+		_, _ = context.TaskList().DeleteTaskByID(task.ID)
 		if err != nil {
 			AbortWithJSONError(c, retValue.Code, err)
 			return
@@ -282,11 +286,11 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	// filter packages by version
 	if c.Request.URL.Query().Get("maximumVersion") == "1" {
 		list.PrepareIndex()
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
 			if err != nil {
 				fmt.Println("filter packages by version, query string parse err: ", err)
-				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+				_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
 			} else {
 				tmpList, err := list.Filter(deb.FilterOptions{
 					Queries: []deb.PackageQuery{versionQ},
@@ -294,15 +298,15 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 
 				if err == nil {
 					if tmpList.Len() > 0 {
-						tmpList.ForEach(func(tp *deb.Package) error {
+						_ = tmpList.ForEach(func(tp *deb.Package) error {
 							list.Remove(tp)
 							return nil
 						})
-						list.Add(p)
+						_ = list.Add(p)
 					}
 				} else {
 					fmt.Println("filter packages by version, filter err: ", err)
-					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+					_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
 				}
 			}
 
@@ -311,7 +315,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -322,7 +326,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
 	}
 }
 
-func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+func AbortWithJSONError(c *gin.Context, code int, err error) {
 	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
-	return c.AbortWithError(code, err)
+	_ = c.AbortWithError(code, err)
 }

api/api_test.go

@@ -24,14 +24,14 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ApiSuite struct {
+type APISuite struct {
 	context    *ctx.AptlyContext
 	flags      *flag.FlagSet
 	configFile *os.File
 	router     http.Handler
 }
 
-var _ = Suite(&ApiSuite{})
+var _ = Suite(&APISuite{})
 
 func createTestConfig() *os.File {
 	file, err := os.CreateTemp("", "aptly")
@@ -45,11 +45,11 @@ func createTestConfig() *os.File {
 	if err != nil {
 		return nil
 	}
-	file.Write(jsonString)
+	_, _ = file.Write(jsonString)
 	return file
 }
 
-func (s *ApiSuite) setupContext() error {
+func (s *APISuite) setupContext() error {
 	aptly.Version = "testVersion"
 	file := createTestConfig()
 	if nil == file {
@@ -75,23 +75,23 @@ func (s *ApiSuite) setupContext() error {
 	return nil
 }
 
-func (s *ApiSuite) SetUpSuite(c *C) {
+func (s *APISuite) SetUpSuite(c *C) {
 	err := s.setupContext()
 	c.Assert(err, IsNil)
 }
 
-func (s *ApiSuite) TearDownSuite(c *C) {
-	os.Remove(s.configFile.Name())
+func (s *APISuite) TearDownSuite(c *C) {
+	_ = os.Remove(s.configFile.Name())
 	s.context.Shutdown()
 }
 
-func (s *ApiSuite) SetUpTest(c *C) {
+func (s *APISuite) SetUpTest(c *C) {
 }
 
-func (s *ApiSuite) TearDownTest(c *C) {
+func (s *APISuite) TearDownTest(c *C) {
 }
 
-func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
 	w := httptest.NewRecorder()
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
@@ -102,32 +102,32 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
 	return w, nil
 }
 
-func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+func (s *APISuite) TestGinRunsInReleaseMode(c *C) {
 	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
 }
 
-func (s *ApiSuite) TestGetVersion(c *C) {
+func (s *APISuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
 }
 
-func (s *ApiSuite) TestGetReadiness(c *C) {
+func (s *APISuite) TestGetReadiness(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/ready", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
 }
 
-func (s *ApiSuite) TestGetHealthiness(c *C) {
+func (s *APISuite) TestGetHealthiness(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
 }
 
-func (s *ApiSuite) TestGetMetrics(c *C) {
+func (s *APISuite) TestGetMetrics(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
@@ -141,7 +141,7 @@ func (s *ApiSuite) TestGetMetrics(c *C) {
 	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
 }
 
-func (s *ApiSuite) TestRepoCreate(c *C) {
+func (s *APISuite) TestRepoCreate(c *C) {
 	body, err := json.Marshal(gin.H{
 		"Name": "dummy",
 	})
@@ -150,7 +150,7 @@ func (s *ApiSuite) TestRepoCreate(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *ApiSuite) TestTruthy(c *C) {
+func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy("no"), Equals, false)
 	c.Check(truthy("n"), Equals, false)
 	c.Check(truthy("off"), Equals, false)

api/db.go

@@ -21,7 +21,7 @@ import (
 // @Success 200 {object} string "Output"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/db/cleanup [post]
-func apiDbCleanup(c *gin.Context) {
+func apiDBCleanup(c *gin.Context) {
 	resources := []string{string(task.AllResourcesKey)}
 	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 		var err error
@@ -109,8 +109,8 @@ func apiDbCleanup(c *gin.Context) {
 
 		if toDelete.Len() > 0 {
 			batch := db.CreateBatch()
-			toDelete.ForEach(func(ref []byte) error {
-				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+			_ = toDelete.ForEach(func(ref []byte) error {
+				_ = collectionFactory.PackageCollection().DeleteByKey(ref, batch)
 				return nil
 			})
 

api/files.go

@@ -13,6 +13,10 @@ import (
 	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 func verifyPath(path string) bool {
 	path = filepath.Clean(path)
 	for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -114,34 +118,69 @@ func apiFilesUpload(c *gin.Context) {
 	}
 
 	stored := []string{}
+	openFiles := []*os.File{}
 
+	// Write all files first
 	for _, files := range c.Request.MultipartForm.File {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer src.Close()
 
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
+				_ = src.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
+			_ = src.Close()
 			if err != nil {
+				_ = dst.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
 
+			// Keep file open for batch sync
+			openFiles = append(openFiles, dst)
 			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
 		}
 	}
 
+	// Sync all files at once to catch ENOSPC errors
+	for i, dst := range openFiles {
+		err := syncFile(dst)
+		if err != nil {
+			// Close all files
+			for _, f := range openFiles {
+				_ = f.Close()
+			}
+			AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
+			return
+		}
+	}
+
+	// Close all files
+	for _, dst := range openFiles {
+		_ = dst.Close()
+	}
+
 	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
 }

api/files_test.go

@@ -0,0 +1,476 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+type FilesUploadDiskFullSuite struct {
+	aptlyContext *ctx.AptlyContext
+	flags        *flag.FlagSet
+	configFile   *os.File
+	router       http.Handler
+}
+
+var _ = Suite(&FilesUploadDiskFullSuite{})
+
+func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
+	aptly.Version = "testVersion"
+
+	file, err := os.CreateTemp("", "aptly")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+		"rootDir":       c.MkDir(),
+	})
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+	_ = file.Close()
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	aptlyContext, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.aptlyContext = aptlyContext
+	s.router = Router(aptlyContext)
+	context = aptlyContext
+}
+
+func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.aptlyContext != nil {
+		s.aptlyContext.Shutdown()
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
+	testContent := []byte("test file content for upload")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "testfile.txt")
+	c.Assert(err, IsNil)
+
+	_, err = part.Write(testContent)
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
+	testContent := bytes.Repeat([]byte("A"), 10000)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "largefile.bin")
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(part, bytes.NewReader(testContent))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir2", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(len(content), Equals, len(testContent))
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
+	testFiles := map[string][]byte{
+		"file1.txt": []byte("content of file 1"),
+		"file2.txt": bytes.Repeat([]byte("B"), 5000),
+		"file3.deb": []byte("debian package content"),
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	for filename, content := range testFiles {
+		part, err := writer.CreateFormFile("file", filename)
+		c.Assert(err, IsNil)
+		_, err = part.Write(content)
+		c.Assert(err, IsNil)
+	}
+
+	err := writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/multitest", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
+	for filename, expectedContent := range testFiles {
+		uploadedFile := filepath.Join(uploadDir, filename)
+		content, err := os.ReadFile(uploadedFile)
+		c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
+		c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
+	oldSyncFile := syncFile
+	syncFile = func(f *os.File) error {
+		if filepath.Base(f.Name()) == "syncfail.txt" {
+			return syscall.ENOSPC
+		}
+		return nil
+	}
+	defer func() { syncFile = oldSyncFile }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part1, err := writer.CreateFormFile("file", "ok.txt")
+	c.Assert(err, IsNil)
+	_, err = part1.Write([]byte("ok"))
+	c.Assert(err, IsNil)
+
+	part2, err := writer.CreateFormFile("file", "syncfail.txt")
+	c.Assert(err, IsNil)
+	_, err = part2.Write([]byte("will fail on sync"))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+	c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
+	c.Check(verifyPath("a/b/c"), Equals, true)
+	c.Check(verifyPath("../x"), Equals, false)
+	c.Check(verifyPath("./x"), Equals, true)
+	c.Check(verifyPath(".."), Equals, false)
+	c.Check(verifyPath("."), Equals, false)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
+	uploadRoot := s.aptlyContext.UploadPath()
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "d1"), Equals, true)
+	c.Check(strings.Contains(body, "d2"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
+	req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 404)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files/dir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "a.txt"), Equals, true)
+	c.Check(strings.Contains(body, "b.txt"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(base)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(filepath.Join(base, "a.txt"))
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
+	req, err := http.NewRequest("DELETE", "/api/files/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.Chmod(base, 0), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	req, err := http.NewRequest("GET", "/api/files/noperms", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	subdir := filepath.Join(base, "subdir")
+	c.Assert(os.MkdirAll(subdir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
+	req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/..", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
+	// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
+	form := &multipart.Form{
+		File: map[string][]*multipart.FileHeader{
+			"file": {{Filename: "broken.bin"}},
+		},
+	}
+
+	req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
+	c.Assert(err, IsNil)
+	req.MultipartForm = form
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.Chmod(base, 0555), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/readonly", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
+	parent := s.aptlyContext.UploadPath()
+	base := filepath.Join(parent, "cantremove")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	c.Assert(os.Chmod(parent, 0555), IsNil)
+	defer func() { _ = os.Chmod(parent, 0777) }()
+
+	req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}

api/gpg.go

@@ -13,26 +13,34 @@ import (
 )
 
 type gpgAddKeyParams struct {
-	// Keyserver, when downloading GpgKeyIDs
-	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
-	// GpgKeyIDs to download from Keyserver, comma separated list
-	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
-	// Armored gpg public ket, instead of downloading from keyserver
-	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
 	// Keyring for adding the keys (default: trustedkeys.gpg)
 	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Add ASCII armored gpg public key, do not download from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+
+	// Keyserver to download keys provided in `GpgKeyID`
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// Keys do download from `Keyserver`, separated by space
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500 8B48AD6246925553"`
 }
 
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
-// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description Add GPG public keys for verifying remote repositories for mirroring.
+// @Description
+// @Description Keys can be added in two ways:
+// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
+// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
+// @Description
 // @Tags Mirrors
+// @Consume  json
+// @Param request body gpgAddKeyParams true "Parameters"
 // @Produce json
 // @Success 200 {object} string "OK"
 // @Failure 400 {object} Error "Bad Request"
-// @Failure 404 {object} Error "Not Found"
-// @Router /api/gpg [post]
+// @Router /api/gpg/key [post]
 func apiGPGAddKey(c *gin.Context) {
 	b := gpgAddKeyParams{}
 	if c.Bind(&b) != nil {
@@ -60,7 +68,7 @@ func apiGPGAddKey(c *gin.Context) {
 			AbortWithJSONError(c, 400, err)
 			return
 		}
-		defer os.RemoveAll(tempdir)
+		defer func() { _ = os.RemoveAll(tempdir) }()
 
 		keypath := filepath.Join(tempdir, "key")
 		keyfile, e := os.Create(keypath)

api/middleware_test.go

@@ -67,17 +67,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "warn")
@@ -130,17 +130,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "info")
@@ -153,17 +153,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "error")
@@ -176,17 +176,17 @@ func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	fmt.Println(capturedOutput)
 

api/mirror.go

@@ -43,7 +43,7 @@ func apiMirrorsList(c *gin.Context) {
 	collection := collectionFactory.RemoteRepoCollection()
 
 	result := []*deb.RemoteRepo{}
-	collection.ForEach(func(repo *deb.RemoteRepo) error {
+	_ = collection.ForEach(func(repo *deb.RemoteRepo) error {
 		result = append(result, repo)
 		return nil
 	})
@@ -175,9 +175,9 @@ func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	mirrorCollection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
@@ -187,21 +187,34 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+			// Fresh checks with current collections
+			snapshots := taskSnapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
-		err = mirrorCollection.Drop(repo)
+		err = taskMirrorCollection.Drop(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
@@ -232,6 +245,7 @@ func apiMirrorsShow(c *gin.Context) {
 	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
 	}
 
 	c.JSON(200, repo)
@@ -319,7 +333,7 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -333,26 +347,8 @@ func apiMirrorsPackages(c *gin.Context) {
 type mirrorUpdateParams struct {
 	// Change mirror name to `Name`
 	Name string `                 json:"Name"                   example:"mirror1"`
-	// Url of the archive to mirror
-	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
-	// Package query that is applied to mirror packages
-	Filter string `               json:"Filter"                 example:"xserver-xorg"`
-	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
-	Architectures []string `      json:"Architectures"          example:"amd64"`
-	// Components to mirror, if not specified aptly would fetch all components
-	Components []string `         json:"Components"             example:"main"`
-	// Gpg keyring(s) for verifing Release file
+	// Gpg keyring(s) for verifying Release file
 	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
-	// Set "true" to include dependencies of matching packages when filtering
-	FilterWithDeps bool `         json:"FilterWithDeps"`
-	// Set "true" to mirror source packages
-	DownloadSources bool `        json:"DownloadSources"`
-	// Set "true" to mirror udeb files
-	DownloadUdebs bool `          json:"DownloadUdebs"`
-	// Set "true" to skip checking if the given components are in the Release file
-	SkipComponentCheck bool `     json:"SkipComponentCheck"`
-	// Set "true" to skip checking if the given architectures are in the Release file
-	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
 	// Set "true" to ignore checksum errors
 	IgnoreChecksums bool `        json:"IgnoreChecksums"`
 	// Set "true" to skip the verification of Release file signatures
@@ -387,21 +383,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
-	remote, err = collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	remote, err = collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	b.Name = remote.Name
-	b.DownloadUdebs = remote.DownloadUdebs
-	b.DownloadSources = remote.DownloadSources
-	b.SkipComponentCheck = remote.SkipComponentCheck
-	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
-	b.FilterWithDeps = remote.FilterWithDeps
-	b.Filter = remote.Filter
-	b.Architectures = remote.Architectures
-	b.Components = remote.Components
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
 	log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -410,6 +399,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Pre-task validation of new name if provided
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
@@ -418,27 +408,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
-	if b.DownloadUdebs != remote.DownloadUdebs {
-		if remote.IsFlat() && b.DownloadUdebs {
-			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
-			return
-		}
-	}
-
-	if b.ArchiveURL != "" {
-		remote.SetArchiveRoot(b.ArchiveURL)
-	}
-
-	remote.Name = b.Name
-	remote.DownloadUdebs = b.DownloadUdebs
-	remote.DownloadSources = b.DownloadSources
-	remote.SkipComponentCheck = b.SkipComponentCheck
-	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
-	remote.FilterWithDeps = b.FilterWithDeps
-	remote.Filter = b.Filter
-	remote.Architectures = b.Architectures
-	remote.Components = b.Components
-
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -447,9 +416,26 @@ func apiMirrorsUpdate(c *gin.Context) {
 
 	resources := []string{string(remote.Key())}
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.RemoteRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		remote, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Fresh rename check inside lock (if renaming)
+		if b.Name != remote.Name {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: mirror %s already exists", b.Name)
+			}
+		}
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
+		err = remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -461,7 +447,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, taskCollectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -480,8 +466,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(),
+			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -491,12 +477,12 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				collection.Update(remote)
+				_ = taskCollection.Update(remote)
 			}
 		}()
 
 		remote.MarkAsUpdating()
-		err = collection.Update(remote)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -579,7 +565,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 							file, e = os.CreateTemp("", task.File.Filename)
 							if e == nil {
 								task.TempDownPath = file.Name()
-								file.Close()
+								_ = file.Close()
 							}
 						}
 						if e != nil {
@@ -600,7 +586,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 						}
 
 						// and import it back to the pool
-						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, taskCollectionFactory.ChecksumCollection(nil))
 						if err != nil {
 							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
 							pushError(err)
@@ -653,8 +639,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		log.Info().Msgf("%s: Finalizing download...", b.Name)
-		remote.FinalizeDownload(collectionFactory, out)
-		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		_ = remote.FinalizeDownload(taskCollectionFactory, out)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

api/mirror_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 type MirrorSuite struct {
-	ApiSuite
+	APISuite
 }
 
 var _ = Suite(&MirrorSuite{})

api/packages_test.go

@@ -5,7 +5,7 @@ import (
 )
 
 type PackagesSuite struct {
-	ApiSuite
+	APISuite
 }
 
 var _ = Suite(&PackagesSuite{})

api/publish.go

@@ -16,8 +16,8 @@ import (
 type signingParams struct {
 	// Don't sign published repository
 	Skip bool `                json:"Skip"           example:"false"`
-	// GPG key ID to use when signing the release, if not specified default key is used
-	GpgKey string `            json:"GpgKey"         example:"A0546A43624A8331"`
+	// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used
+	GpgKey string `            json:"GpgKey"         example:"KEY_ID_a, KEY_ID_b"`
 	// GPG keyring to use (instead of default)
 	Keyring string `           json:"Keyring"        example:"trustedkeys.gpg"`
 	// GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -41,7 +41,21 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(options.GpgKey)
+
+	var multiGpgKeys []string
+	// REST params have priority over config
+	if options.GpgKey != "" {
+		for _, p := range strings.Split(options.GpgKey, ",") {
+			if t := strings.TrimSpace(p); t != "" {
+				multiGpgKeys = append(multiGpgKeys, t)
+			}
+		}
+	} else if len(context.Config().GpgKeys) > 0 {
+		multiGpgKeys = context.Config().GpgKeys
+	}
+	for _, gpgKey := range multiGpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
 
@@ -110,7 +124,7 @@ func apiPublishList(c *gin.Context) {
 // @Description See also: `aptly publish show`
 // @Tags Publish
 // @Produce json
-// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs"
+// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs"
 // @Param distribution path string true "distribution name"
 // @Success 200 {object} deb.PublishedRepo
 // @Failure 404 {object} Error "Published repository not found"
@@ -146,10 +160,6 @@ type publishedRepoCreateParams struct {
 	Sources []sourceParams `binding:"required"    json:"Sources"`
 	// Distribution name, if missing Aptly would try to guess from sources
 	Distribution string `                         json:"Distribution"          example:"bookworm"`
-	// Value of Label: field in published repository stanza
-	Label string `                                json:"Label"                 example:""`
-	// Value of Origin: field in published repository stanza
-	Origin string `                               json:"Origin"                example:""`
 	// when publishing, overwrite files in pool/ directory without notice
 	ForceOverwrite bool `                         json:"ForceOverwrite"        example:"false"`
 	// Override list of published architectures
@@ -182,7 +192,7 @@ type publishedRepoCreateParams struct {
 // @Description **Example:**
 // @Description ```
 // @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos
-// @Description {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
+// @Description {"Architectures":["i386"],"Distribution":"wheezy","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
 // @Description ```
 // @Description
 // @Description See also: `aptly publish create`
@@ -249,7 +259,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 				return
 			}
 
-			resources = append(resources, string(snapshot.ResourceKey()))
+			resources = append(resources, string(snapshot.Key()))
 			sources = append(sources, snapshot)
 		}
 	} else if b.SourceKind == deb.SourceLocalRepo {
@@ -280,11 +290,24 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		multiDist = *b.MultiDist
 	}
 
-	collection := collectionFactory.PublishedRepoCollection()
+	// Non-MultiDist publishes share a single pool/ directory under the
+	// prefix.  Lock at the prefix level so that concurrent publish/drop
+	// operations on sibling distributions cannot race during cleanup.
+	if !multiDist {
+		storagePrefix := prefix
+		if storage != "" {
+			storagePrefix = storage + ":" + prefix
+		}
+
+		resources = append(resources, deb.PrefixPoolLockKey(storagePrefix))
+	}
 
 	taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"",
 		b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`))
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
 		taskDetail := task.PublishDetail{
 			Detail: detail,
 		}
@@ -296,10 +319,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		for _, source := range sources {
 			switch s := source.(type) {
 			case *deb.Snapshot:
-				snapshotCollection := collectionFactory.SnapshotCollection()
+				snapshotCollection := taskCollectionFactory.SnapshotCollection()
 				err = snapshotCollection.LoadComplete(s)
 			case *deb.LocalRepo:
-				localCollection := collectionFactory.LocalRepoCollection()
+				localCollection := taskCollectionFactory.LocalRepoCollection()
 				err = localCollection.LoadComplete(s)
 			default:
 				err = fmt.Errorf("unexpected type for source: %T", source)
@@ -309,23 +332,17 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			}
 		}
 
-		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist)
+		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, taskCollectionFactory, multiDist)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		resources = append(resources, string(published.Key()))
-
-		if b.Origin != "" {
-			published.Origin = b.Origin
-		}
 		if b.NotAutomatic != "" {
 			published.NotAutomatic = b.NotAutomatic
 		}
 		if b.ButAutomaticUpgrades != "" {
 			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
 		}
-		published.Label = b.Label
 
 		published.SkipContents = context.Config().SkipContentsPublishing
 		if b.SkipContents != nil {
@@ -341,18 +358,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			published.AcquireByHash = *b.AcquireByHash
 		}
 
-		duplicate := collection.CheckDuplicate(published)
+		duplicate := taskCollection.CheckDuplicate(published)
 		if duplicate != nil {
-			collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			_ = taskCollectionFactory.PublishedRepoCollection().LoadComplete(duplicate, taskCollectionFactory)
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		err = collection.Add(published)
+		err = taskCollection.Add(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -393,7 +410,6 @@ type publishedRepoUpdateSwitchParams struct {
 // @Description
 // @Description See also: `aptly publish update` / `aptly publish switch`
 // @Tags Publish
-// @Produce json
 // @Param prefix path string true "publishing prefix"
 // @Param distribution path string true "distribution name"
 // @Param _async query bool false "Run in background and return task object"
@@ -425,6 +441,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
@@ -432,46 +449,74 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
+	resources := []string{string(published.Key())}
+
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
-	} else if published.SourceKind == deb.SourceSnapshot {
-		for _, snapshotInfo := range b.Snapshots {
-			_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
 			if err2 != nil {
 				AbortWithJSONError(c, http.StatusNotFound, err2)
 				return
 			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, snapshotInfo := range b.Snapshots {
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
 		}
 	} else {
 		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
 	}
 
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
-	resources := []string{string(published.Key())}
+	// Field mutations and fresh DB load are deferred to inside the task so
+	// they always operate on a consistent state after the lock is held.
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(published, collectionFactory)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
 		}
 
 		revision := published.ObtainRevision()
@@ -485,17 +530,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			}
 		}
 
-		result, err := published.Update(collectionFactory, out)
+		result, err := published.Update(taskCollectionFactory, out)
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -503,10 +548,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
@@ -551,10 +605,19 @@ func apiPublishDrop(c *gin.Context) {
 	}
 
 	resources := []string{string(published.Key())}
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that a drop cannot race
+	// with a concurrent update or drop of a sibling distribution during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
 	taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.Remove(context, storage, prefix, distribution,
-			collectionFactory, out, force, skipCleanup)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		err := taskCollection.Remove(context, storage, prefix, distribution,
+			taskCollectionFactory, out, force, skipCleanup)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
 		}
@@ -590,43 +653,52 @@ func apiPublishAddSource(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly (no LoadComplete) to verify existence and obtain the
+	// resource key and task name.  The actual mutation is performed inside
+	// the task on a freshly loaded copy to prevent lost-update races.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	component := b.Component
-	name := b.Name
-
-	_, exists := sources[component]
-	if exists {
-		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component))
-		return
-	}
-
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		component := b.Component
+		name := b.Name
+
+		_, exists := sources[component]
+		if exists {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("unable to create: Component '%s' already exists", component)
+		}
+
+		sources[component] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -708,39 +780,48 @@ func apiPublishSetSources(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := make(map[string]string, len(b))
-	revision.Sources = sources
-
-	for _, source := range b {
-		component := source.Component
-		name := source.Name
-		sources[component] = name
-	}
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := make(map[string]string, len(b))
+		revision.Sources = sources
+
+		for _, source := range b {
+			component := source.Component
+			name := source.Name
+			sources[component] = name
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -773,24 +854,33 @@ func apiPublishDropChanges(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and DropRevision happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	published.DropRevision()
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		published.DropRevision()
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -826,51 +916,58 @@ func apiPublishUpdateSource(c *gin.Context) {
 	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
-	component := slashEscape(c.Params.ByName("component"))
+	urlComponent := slashEscape(c.Params.ByName("component"))
+
+	// Default component to the URL path segment; the body may rename it.
+	b.Component = urlComponent
+	if c.Bind(&b) != nil {
+		return
+	}
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component))
-		return
-	}
-
-	b.Component = component
-	b.Name = revision.Sources[component]
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	if b.Component != component {
-		delete(sources, component)
-	}
-
-	component = b.Component
-	name := b.Name
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[urlComponent]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: Component '%s' does not exist", urlComponent)
+		}
+
+		if b.Component != urlComponent {
+			delete(sources, urlComponent)
+		}
+
+		newComponent := b.Component
+		name := b.Name
+		sources[newComponent] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -907,33 +1004,41 @@ func apiPublishRemoveSource(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component))
-		return
-	}
-
-	delete(sources, component)
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[component]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: Component '%s' does not exist", component)
+		}
+
+		delete(sources, component)
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -997,48 +1102,92 @@ func apiPublishUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and field mutations happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
-	}
-
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
 	resources := []string{string(published.Key())}
+
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
+
+	// Lock source repos / snapshots the same way apiPublishUpdateSwitch does,
+	// because published.Update() reads from them and concurrent modification
+	// would produce an inconsistent view.
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, uuid := range published.Sources {
+			snapshot, err2 := snapshotCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
+		}
+	}
+
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		result, err := published.Update(collectionFactory, out)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
+		}
+
+		result, err := published.Update(taskCollectionFactory, out)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -1046,10 +1195,19 @@ func apiPublishUpdate(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil

api/published_file_missing_test.go

@@ -0,0 +1,737 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+// PublishedFileMissingSuite reproduces the exact bug where:
+// - Package import succeeds
+// - Metadata is updated (Packages.gz shows the package)
+// - Publish reports success
+// - BUT the .deb file is missing from the published pool directory
+// - Result: apt-get returns 404 when trying to download the package
+type PublishedFileMissingSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+	tempDir    string
+	poolPath   string
+	publicPath string
+}
+
+var _ = Suite(&PublishedFileMissingSuite{})
+
+func (s *PublishedFileMissingSuite) SetUpSuite(c *C) {
+	aptly.Version = "publishedFileMissingTest"
+
+	tempDir, err := os.MkdirTemp("", "aptly-published-missing-test")
+	c.Assert(err, IsNil)
+	s.tempDir = tempDir
+	s.poolPath = filepath.Join(tempDir, "pool")
+	s.publicPath = filepath.Join(tempDir, "public")
+
+	file, err := os.CreateTemp("", "aptly-published-missing-config")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	config := gin.H{
+		"rootDir":                    tempDir,
+		"downloadDir":                filepath.Join(tempDir, "download"),
+		"architectures":              []string{"amd64"},
+		"dependencyFollowSuggests":   false,
+		"dependencyFollowRecommends": false,
+		"gpgDisableSign":             true,
+		"gpgDisableVerify":           true,
+		"gpgProvider":                "internal",
+		"skipLegacyPool":             true,
+		"enableMetricsEndpoint":      false,
+	}
+
+	jsonString, err := json.Marshal(config)
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+
+	flags := flag.NewFlagSet("publishedFileMissingTestFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", true, "disable database locking for test")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "config file")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *PublishedFileMissingSuite) TearDownSuite(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.context != nil {
+		s.context.Shutdown()
+	}
+	if s.tempDir != "" {
+		_ = os.RemoveAll(s.tempDir)
+	}
+}
+
+func (s *PublishedFileMissingSuite) SetUpTest(c *C) {
+	collectionFactory := s.context.NewCollectionFactory()
+
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+	_ = localRepoCollection.ForEach(func(repo *deb.LocalRepo) error {
+		_ = localRepoCollection.Drop(repo)
+		return nil
+	})
+
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+	_ = publishedCollection.ForEach(func(published *deb.PublishedRepo) error {
+		_ = publishedCollection.Remove(s.context, published.Storage, published.Prefix,
+			published.Distribution, collectionFactory, nil, true, true)
+		return nil
+	})
+}
+
+func (s *PublishedFileMissingSuite) TearDownTest(c *C) {
+	s.SetUpTest(c)
+}
+
+func (s *PublishedFileMissingSuite) httpRequest(c *C, method string, url string, body []byte) *httptest.ResponseRecorder {
+	w := httptest.NewRecorder()
+	var req *http.Request
+	var err error
+
+	if body != nil {
+		req, err = http.NewRequest(method, url, bytes.NewReader(body))
+	} else {
+		req, err = http.NewRequest(method, url, nil)
+	}
+	c.Assert(err, IsNil)
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w
+}
+
+func (s *PublishedFileMissingSuite) createDebPackage(c *C, uploadID, packageName, version string) {
+	uploadPath := s.context.UploadPath()
+	uploadDir := filepath.Join(uploadPath, uploadID)
+	err := os.MkdirAll(uploadDir, 0755)
+	c.Assert(err, IsNil)
+
+	tempDir, err := os.MkdirTemp("", "deb-build")
+	c.Assert(err, IsNil)
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	debianDir := filepath.Join(tempDir, "DEBIAN")
+	err = os.MkdirAll(debianDir, 0755)
+	c.Assert(err, IsNil)
+
+	controlContent := fmt.Sprintf(`Package: %s
+Version: %s
+Section: libs
+Priority: optional
+Architecture: amd64
+Maintainer: Test <test@example.com>
+Description: Test package
+ Test package for published file missing bug.
+`, packageName, version)
+
+	err = os.WriteFile(filepath.Join(debianDir, "control"), []byte(controlContent), 0644)
+	c.Assert(err, IsNil)
+
+	usrDir := filepath.Join(tempDir, "usr", "lib")
+	err = os.MkdirAll(usrDir, 0755)
+	c.Assert(err, IsNil)
+	err = os.WriteFile(filepath.Join(usrDir, "lib.so"), []byte("library"), 0644)
+	c.Assert(err, IsNil)
+
+	debFile := filepath.Join(uploadDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+	cmd := exec.Command("dpkg-deb", "--build", tempDir, debFile)
+	err = cmd.Run()
+	c.Assert(err, IsNil)
+}
+
+// TestPublishedFileGoMissing reproduces the exact production bug
+func (s *PublishedFileMissingSuite) TestPublishedFileGoMissing(c *C) {
+	c.Log("=== Reproducing: Package in metadata but 404 on download ===")
+
+	// Create and publish a repository
+	repoName := "test-repo"
+	distribution := "bullseye"
+
+	createBody, _ := json.Marshal(gin.H{
+		"Name":                repoName,
+		"DefaultDistribution": distribution,
+		"DefaultComponent":    "main",
+	})
+	resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to create repo: %s", resp.Body.String()))
+
+	publishBody, _ := json.Marshal(gin.H{
+		"SourceKind":    "local",
+		"Distribution":  distribution,
+		"Architectures": []string{"amd64"},
+		"Sources": []gin.H{
+			{"Component": "main", "Name": repoName},
+		},
+		"Signing": gin.H{"Skip": true},
+	})
+	resp = s.httpRequest(c, "POST", "/api/publish/hrt", publishBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to publish: %s", resp.Body.String()))
+
+	// Create package
+	packageName := "hrt-libblobbyclient1"
+	version := "20250926.152427+hrtdeb11"
+	uploadID := "test-upload-1"
+
+	s.createDebPackage(c, uploadID, packageName, version)
+
+	// Add package
+	resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package: %s", resp.Body.String()))
+
+	// Update publish
+	updateBody, _ := json.Marshal(gin.H{
+		"Signing":        gin.H{"Skip": true},
+		"ForceOverwrite": true,
+	})
+	resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/hrt/%s", distribution), updateBody)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String()))
+
+	// Now check if the file is actually accessible in the published location
+	publishedStorage, err := s.context.GetPublishedStorage("")
+	c.Assert(err, IsNil)
+	publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+	// Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb
+	expectedPath := filepath.Join(publicPath, "hrt", "pool", "main", "h", packageName,
+		fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+	c.Logf("Checking for published file at: %s", expectedPath)
+
+	fileInfo, err := os.Stat(expectedPath)
+	fileExists := err == nil
+
+	c.Logf("File exists: %v", fileExists)
+	if fileExists {
+		c.Logf("File size: %d bytes", fileInfo.Size())
+	}
+
+	// Check metadata
+	resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+	var packages []string
+	err = json.Unmarshal(resp.Body.Bytes(), &packages)
+	c.Assert(err, IsNil)
+	c.Logf("Packages in metadata: %d", len(packages))
+
+	// THE BUG: Metadata says package exists, but file is missing from published location
+	if len(packages) > 0 && !fileExists {
+		c.Logf("★★★ BUG REPRODUCED! ★★★")
+		c.Logf("Metadata shows %d package(s) but file is missing at: %s", len(packages), expectedPath)
+		c.Logf("This is exactly what causes: 404 Not Found [IP: 10.20.72.62 3142]")
+
+		c.Fatal("BUG CONFIRMED: Package in metadata but missing from published directory!")
+	}
+
+	c.Assert(fileExists, Equals, true, Commentf(
+		"Published file should exist at %s when package is in metadata", expectedPath))
+}
+
+// TestConcurrentPublishRace tries to trigger the race with concurrent publishes
+func (s *PublishedFileMissingSuite) TestConcurrentPublishRace(c *C) {
+	c.Log("=== Testing concurrent publish race condition ===")
+
+	const numIterations = 4
+
+	for iteration := 0; iteration < numIterations; iteration++ {
+		c.Logf("--- Iteration %d/%d ---", iteration+1, numIterations)
+
+		// Create repo
+		repoName := fmt.Sprintf("race-repo-%d", iteration)
+		distribution := fmt.Sprintf("dist-%d", iteration)
+
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoName,
+			"DefaultDistribution": distribution,
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		publishBody, _ := json.Marshal(gin.H{
+			"SourceKind":    "local",
+			"Distribution":  distribution,
+			"Architectures": []string{"amd64"},
+			"Sources": []gin.H{
+				{"Component": "main", "Name": repoName},
+			},
+			"Signing": gin.H{"Skip": true},
+		})
+		resp = s.httpRequest(c, "POST", "/api/publish/concurrent", publishBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		// Create multiple packages
+		var wg sync.WaitGroup
+		numPackages := 5
+
+		for i := 0; i < numPackages; i++ {
+			wg.Add(1)
+			go func(idx int) {
+				defer wg.Done()
+
+				packageName := fmt.Sprintf("pkg-%d-%d", iteration, idx)
+				version := "1.0.0"
+				uploadID := fmt.Sprintf("upload-%d-%d", iteration, idx)
+
+				s.createDebPackage(c, uploadID, packageName, version)
+
+				// Add package
+				resp := s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+				c.Logf("Package %d add: %d", idx, resp.Code)
+
+				// Small delay
+				time.Sleep(time.Duration(5+idx*2) * time.Millisecond)
+
+				// Publish
+				updateBody, _ := json.Marshal(gin.H{
+					"Signing":        gin.H{"Skip": true},
+					"ForceOverwrite": true,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/concurrent/%s", distribution), updateBody)
+				c.Logf("Publish %d: %d", idx, resp.Code)
+			}(i)
+		}
+
+		wg.Wait()
+		time.Sleep(100 * time.Millisecond)
+
+		// Check all packages
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+		var packages []string
+		err := json.Unmarshal(resp.Body.Bytes(), &packages)
+		c.Assert(err, IsNil)
+
+		// Check published files
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("pkg-%d-%d", iteration, i)
+			version := "1.0.0"
+
+			// Calculate pool path
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, "concurrent", "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, expectedPath)
+			}
+		}
+
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG DETECTED in iteration %d/%d! ★★★", iteration+1, numIterations)
+			c.Logf("Metadata shows %d packages, but %d files are MISSING:", len(packages), len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  [iter %d] File MISSING %d/%d: %s", iteration+1, i+1, len(missingFiles), f)
+			}
+
+			c.Fatalf("BUG REPRODUCED in iteration %d/%d: %d published files missing", iteration+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iteration+1, numIterations, numPackages)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced with current timing", numIterations)
+}
+
+// TestIdenticalPackageRace tests the specific case of identical SHA256 packages
+func (s *PublishedFileMissingSuite) TestIdenticalPackageRace(c *C) {
+	c.Log("=== AGGRESSIVE test: identical package (same SHA256) race ===")
+
+	const numIterations = 4
+	packageName := "shared-package"
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("Iteration %d/%d", iter+1, numIterations)
+
+		// Create two repos that will get the SAME package (unique per iteration)
+		repos := []string{fmt.Sprintf("identical-a-%d", iter), fmt.Sprintf("identical-b-%d", iter)}
+		dists := []string{fmt.Sprintf("dist-a-%d", iter), fmt.Sprintf("dist-b-%d", iter)}
+
+		for i := range repos {
+			createBody, _ := json.Marshal(gin.H{
+				"Name":                repos[i],
+				"DefaultDistribution": dists[i],
+				"DefaultComponent":    "main",
+			})
+			resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+			c.Assert(resp.Code, Equals, 201)
+
+			publishBody, _ := json.Marshal(gin.H{
+				"SourceKind":    "local",
+				"Distribution":  dists[i],
+				"Architectures": []string{"amd64"},
+				"Sources": []gin.H{
+					{"Component": "main", "Name": repos[i]},
+				},
+				"Signing": gin.H{"Skip": true},
+				"SkipBz2": true,
+			})
+			resp = s.httpRequest(c, "POST", "/api/publish/identical", publishBody)
+			c.Assert(resp.Code, Equals, 201)
+		}
+
+		// Create IDENTICAL package file with UNIQUE VERSION per iteration
+		version := fmt.Sprintf("1.0.%d", iter)
+		uploadID1 := fmt.Sprintf("identical-upload-1-%d", iter)
+		uploadID2 := fmt.Sprintf("identical-upload-2-%d", iter)
+
+		s.createDebPackage(c, uploadID1, packageName, version)
+
+		// Copy to second upload (same SHA256)
+		uploadPath := s.context.UploadPath()
+		src := filepath.Join(uploadPath, uploadID1, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+		destDir := filepath.Join(uploadPath, uploadID2)
+		err := os.MkdirAll(destDir, 0755)
+		c.Assert(err, IsNil)
+		dest := filepath.Join(destDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		srcData, readErr := os.ReadFile(src)
+		c.Assert(readErr, IsNil)
+		err = os.WriteFile(dest, srcData, 0644)
+		c.Assert(err, IsNil)
+
+		// Race: add and publish both simultaneously
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[0], uploadID1), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[0]), updateBody)
+		}()
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[1], uploadID2), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[1]), updateBody)
+		}()
+
+		wg.Wait()
+		time.Sleep(200 * time.Millisecond)
+		c.Logf("[iter %d] All operations complete", iter)
+
+		// Check the shared pool location
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		poolSubdir := string(packageName[0])
+		sharedPoolPath := filepath.Join(publicPath, "identical", "pool", "main", poolSubdir, packageName,
+			fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		fileInfo, err := os.Stat(sharedPoolPath)
+		fileExists := err == nil
+
+		if fileExists {
+			c.Logf("[iter %d] File EXISTS at %s (size: %d)", iter, sharedPoolPath, fileInfo.Size())
+		} else {
+			c.Logf("[iter %d] File MISSING at %s (error: %v)", iter, sharedPoolPath, err)
+		}
+
+		// Check metadata
+		var packagesA, packagesB []string
+		resp := s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[0]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesA)
+		c.Assert(err, IsNil)
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[1]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesB)
+		c.Assert(err, IsNil)
+
+		c.Logf("[iter %d] Packages in metadata: A=%d, B=%d", iter, len(packagesA), len(packagesB))
+
+		// THE BUG: Both repos show packages in metadata, but the shared pool file is missing
+		if (len(packagesA) > 0 || len(packagesB) > 0) && !fileExists {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d! ★★★", iter+1)
+			c.Logf("Packages in metadata A: %d, B: %d", len(packagesA), len(packagesB))
+			c.Logf("Shared pool file exists: %v", fileExists)
+			c.Logf("Pool path: %s", sharedPoolPath)
+
+			// List what files ARE in the pool directory
+			poolDir := filepath.Dir(sharedPoolPath)
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("Files in pool directory %s:", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s", entry.Name())
+				}
+			}
+
+			c.Fatalf("Metadata shows packages but shared pool file is missing (iteration %d)", iter+1)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced", numIterations)
+}
+
+// TestConcurrentSnapshotPublishToSamePrefix reproduces the EXACT production bug:
+// Multiple snapshots are published concurrently to the SAME prefix but different distributions.
+// Example from production logs:
+//   - trixie-pgdg published to "external/postgres-auto/trixie"
+//   - bullseye-pgdg published to "external/postgres-auto/bullseye"
+// Both share the same pool directory, causing cleanup race conditions.
+func (s *PublishedFileMissingSuite) TestConcurrentSnapshotPublishToSamePrefix(c *C) {
+	const numIterations = 4
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("--- Iteration %d/%d ---", iter+1, numIterations)
+
+		// Create two repos with different packages (simulating trixie-pgdg and bullseye-pgdg)
+		repoTrixie := fmt.Sprintf("trixie-pgdg-%d", iter)
+		repoBullseye := fmt.Sprintf("bullseye-pgdg-%d", iter)
+
+		// Create trixie repo
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoTrixie,
+			"DefaultDistribution": "trixie",
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie repo"))
+
+		// Create bullseye repo
+		createBody, _ = json.Marshal(gin.H{
+			"Name":                repoBullseye,
+			"DefaultDistribution": "bullseye",
+			"DefaultComponent":    "main",
+		})
+		resp = s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye repo"))
+
+		// Add packages to both repos
+		numPackages := 3
+
+		// Add packages to trixie repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("trixie-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoTrixie, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to trixie"))
+		}
+
+		// Add packages to bullseye repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("bullseye-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoBullseye, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to bullseye"))
+		}
+
+		// Create snapshots from both repos
+		snapshotTrixie := fmt.Sprintf("%s-snap", repoTrixie)
+		snapshotBullseye := fmt.Sprintf("%s-snap", repoBullseye)
+
+		createSnapshotBody, _ := json.Marshal(gin.H{"Name": snapshotTrixie})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoTrixie), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie snapshot"))
+
+		createSnapshotBody, _ = json.Marshal(gin.H{"Name": snapshotBullseye})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoBullseye), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye snapshot"))
+
+		// Publish both snapshots CONCURRENTLY to the SAME prefix
+		// This mimics production where both are published to "external/postgres-auto"
+		// Use the SAME prefix across all iterations to trigger the race more aggressively
+		sharedPrefix := "postgres-auto"
+
+		var wg sync.WaitGroup
+		var trixiePublishCode, bullseyePublishCode int
+
+		wg.Add(2)
+
+		// Publish or update trixie snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "trixie",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false, // Force cleanup to run
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE (this is what happens in production)
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/trixie", sharedPrefix), updateBody)
+			}
+			trixiePublishCode = resp.Code
+			c.Logf("[iter %d] Trixie publish/update completed: %d", iter, resp.Code)
+		}()
+
+		// Publish or update bullseye snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "bullseye",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/bullseye", sharedPrefix), updateBody)
+			}
+			bullseyePublishCode = resp.Code
+			c.Logf("[iter %d] Bullseye publish/update completed: %d", iter, resp.Code)
+		}()
+
+		wg.Wait()
+		time.Sleep(50 * time.Millisecond)
+
+		// Verify publishes succeeded (201 for create, 200 for update)
+		expectedCode := 201
+		if iter > 0 {
+			expectedCode = 200
+		}
+		c.Assert(trixiePublishCode, Equals, expectedCode, Commentf("Trixie publish/update should succeed"))
+		c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed"))
+
+		// Verify ALL package files exist in the published pool
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		expectedFiles := []string{}
+
+		// Check trixie packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("TRIXIE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// Check bullseye packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("BULLSEYE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// BUG: Files from one distribution are deleted by the other's cleanup
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d/%d! ★★★", iter+1, numIterations)
+			c.Logf("Both publishes to prefix '%s' succeeded, but %d files are MISSING:", sharedPrefix, len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  Missing file %d/%d: %s", i+1, len(missingFiles), f)
+			}
+
+			c.Logf("\nThis reproduces the exact production bug where:")
+			c.Logf("  1. Mirror updates complete successfully")
+			c.Logf("  2. Snapshots are created")
+			c.Logf("  3. Both snapshots publish to same prefix (different distributions)")
+			c.Logf("  4. Cleanup from one publish DELETES files from the other")
+			c.Logf("  5. Result: apt-get returns 404 when downloading packages")
+
+			// List what's actually in the pool
+			poolDir := filepath.Join(publicPath, sharedPrefix, "pool", "main")
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("\nActual pool directory contents (%s):", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s/", entry.Name())
+				}
+			}
+
+			c.Fatalf("BUG CONFIRMED (iteration %d/%d): %d files missing from shared pool",
+				iter+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iter+1, numIterations, len(expectedFiles))
+		}
+	}
+	c.Logf("✓ All %d iterations passed - no files missing", numIterations)
+}

api/repos.go

@@ -24,19 +24,19 @@ import (
 // @Tags Repos
 // @Produce html
 // @Success 200 {object} string "HTML"
-// @Router /api/repos [get]
+// @Router /repos [get]
 func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 		c.Writer.Flush()
-		c.Writer.WriteString("<pre>\n")
+		_, _ = c.Writer.WriteString("<pre>\n")
 		if len(localRepos) == 0 {
-			c.Writer.WriteString("<a href=\"-/\">default</a>\n")
+			_, _ = c.Writer.WriteString("<a href=\"-/\">default</a>\n")
 		}
 		for publishPrefix := range localRepos {
-			c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
+			_, _ = c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
 		}
-		c.Writer.WriteString("</pre>")
+		_, _ = c.Writer.WriteString("</pre>")
 		c.Writer.Flush()
 	}
 }
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
 // @Param pkgPath path string true "Package Path" allowReserved=true
 // @Produce json
 // @Success 200 ""
-// @Router /api/{storage}/{pkgPath} [get]
+// @Router /repos/{storage}/{pkgPath} [get]
 func reposServeInAPIMode(c *gin.Context) {
 	pkgpath := c.Param("pkgPath")
 
@@ -60,7 +60,12 @@ func reposServeInAPIMode(c *gin.Context) {
 		storage = "filesystem:" + storage
 	}
 
-	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage(storage)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	c.FileFromFS(pkgpath, http.Dir(publicPath))
 }
 
@@ -76,7 +81,7 @@ func apiReposList(c *gin.Context) {
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	collection.ForEach(func(r *deb.LocalRepo) error {
+	_ = collection.ForEach(func(r *deb.LocalRepo) error {
 		result = append(result, r)
 		return nil
 	})
@@ -93,7 +98,7 @@ type repoCreateParams struct {
 	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
 	// Default component when publishing from this local repo
 	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
-	// Snapshot name to create repoitory from (optional)
+	// Snapshot name to create repository from (optional)
 	FromSnapshot string `            json:"FromSnapshot"         example:""`
 }
 
@@ -107,9 +112,9 @@ type repoCreateParams struct {
 // @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
 // @Description ```
 // @Tags Repos
-// @Produce  json
 // @Consume  json
 // @Param request body repoCreateParams true "Parameters"
+// @Produce  json
 // @Success 201 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Source snapshot not found"
 // @Failure 409 {object} Error "Local repo already exists"
@@ -122,64 +127,82 @@ func apiReposCreate(c *gin.Context) {
 		return
 	}
 
-	repo := deb.NewLocalRepo(b.Name, b.Comment)
-	repo.DefaultComponent = b.DefaultComponent
-	repo.DefaultDistribution = b.DefaultDistribution
-
+	// Handler: Pre-task validations (shallow)
 	collectionFactory := context.NewCollectionFactory()
 
+	var resources []string
 	if b.FromSnapshot != "" {
-		var snapshot *deb.Snapshot
-
-		snapshotCollection := collectionFactory.SnapshotCollection()
-
-		snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+		snapshot, err := collectionFactory.SnapshotCollection().ByName(b.FromSnapshot)
 		if err != nil {
 			AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
 			return
 		}
+		resources = append(resources, string(snapshot.Key()))
+	}
 
-		err = snapshotCollection.LoadComplete(snapshot)
-		if err != nil {
-			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
-			return
+	taskName := fmt.Sprintf("Create repository %s", b.Name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection and check/create ATOMIC inside task
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Check duplicate inside lock
+		if _, err := taskCollection.ByName(b.Name); err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+				fmt.Errorf("local repo with name %s already exists", b.Name)
 		}
 
-		repo.UpdateRefList(snapshot.RefList())
-	}
+		// Create repo
+		repo := deb.NewLocalRepo(b.Name, b.Comment)
+		repo.DefaultComponent = b.DefaultComponent
+		repo.DefaultDistribution = b.DefaultDistribution
 
-	localRepoCollection := collectionFactory.LocalRepoCollection()
+		if b.FromSnapshot != "" {
+			snapshotCollection := taskCollectionFactory.SnapshotCollection()
 
-	if _, err := localRepoCollection.ByName(b.Name); err == nil {
-		AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
-		return
-	}
+			snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, 
+					fmt.Errorf("source snapshot not found: %s", err)
+			}
 
-	err := localRepoCollection.Add(repo)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, err)
-		return
-	}
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, 
+					fmt.Errorf("unable to load source snapshot: %s", err)
+			}
 
-	c.JSON(http.StatusCreated, repo)
+			repo.UpdateRefList(snapshot.RefList())
+		}
+
+		err := taskCollection.Add(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: repo}, nil
+	})
 }
 
 type reposEditParams struct {
 	// Name of repository to modify
-	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
+	Name *string `                       json:"Name"                 example:"new-repo-name"`
 	// Change Comment of repository
 	Comment *string `                    json:"Comment"              example:"example repo"`
 	// Change Default Distribution for publishing
 	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
-	// Change Devault Component for publishing
+	// Change Default Component for publishing
 	DefaultComponent *string `        json:"DefaultComponent"     example:""`
 }
 
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name to modify"
+// @Consume  json
 // @Param request body reposEditParams true "Parameters"
+// @Produce json
 // @Success 200 {object} deb.LocalRepo "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 500 {object} Error "Internal Server Error"
@@ -189,50 +212,74 @@ func apiReposEdit(c *gin.Context) {
 	if c.Bind(&b) != nil {
 		return
 	}
-
+	// Load shallowly for 404 check and resource key.
+	// Mutation and duplicate check happen inside the task for atomicity.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
-	if b.Name != nil {
-		_, err := collection.ByName(*b.Name)
-		if err == nil {
-			// already exists
-			AbortWithJSONError(c, 404, err)
+	if b.Name != nil && *b.Name != name {
+		if _, err = collection.ByName(*b.Name); err == nil {
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: local repo %q already exists", *b.Name))
 			return
 		}
-		repo.Name = *b.Name
-	}
-	if b.Comment != nil {
-		repo.Comment = *b.Comment
-	}
-	if b.DefaultDistribution != nil {
-		repo.DefaultDistribution = *b.DefaultDistribution
-	}
-	if b.DefaultComponent != nil {
-		repo.DefaultComponent = *b.DefaultComponent
 	}
 
-	err = collection.Update(repo)
-	if err != nil {
-		AbortWithJSONError(c, 500, err)
-		return
-	}
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Edit repository %s", name)
 
-	c.JSON(200, repo)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		// Check and update ATOMIC (inside lock)
+		if b.Name != nil && *b.Name != name {
+			_, err := taskCollection.ByName(*b.Name)
+			if err == nil {
+				// already exists
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+					fmt.Errorf("local repo with name %q already exists", *b.Name)
+			}
+			repo.Name = *b.Name
+		}
+		if b.Comment != nil {
+			repo.Comment = *b.Comment
+		}
+		if b.DefaultDistribution != nil {
+			repo.DefaultDistribution = *b.DefaultDistribution
+		}
+		if b.DefaultComponent != nil {
+			repo.DefaultComponent = *b.DefaultComponent
+		}
+
+		err = taskCollection.Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
 }
 
 // GET /api/repos/:name
 // @Summary Get Repository Info
 // @Description Returns basic information about local repository.
 // @Tags Repos
-// @Produce  json
 // @Param name path string true "Repository name"
+// @Produce  json
 // @Success 200 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Repository not found"
 // @Router /api/repos/{name} [get]
@@ -254,9 +301,10 @@ func apiReposShow(c *gin.Context) {
 // @Description Cannot drop repos that are published.
 // @Description Needs force=1 to drop repos used as source by other repos.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Param force query int false "force: 1 to enable"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "Repo object"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Repo Conflict"
@@ -265,10 +313,10 @@ func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	name := c.Params.ByName("name")
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full checks (published/snapshots) happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	repo, err := collection.ByName(name)
 	if err != nil {
@@ -279,19 +327,32 @@ func apiReposDrop(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.ByLocalRepo(repo)
+		// Task: Create fresh collections inside task after lock acquired
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Re-read repo with fresh collection after lock
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		// Check with fresh collections
+		published := taskPublishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			snapshots := taskSnapshotCollection.ByLocalRepoSource(repo)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
 			}
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, taskCollection.Drop(repo)
 	})
 }
 
@@ -306,12 +367,12 @@ func apiReposDrop(c *gin.Context) {
 // @Description ["Pi386 aptly 0.8 966561016b44ed80"]
 // @Description ```
 // @Tags Repos
-// @Produce json
-// @Param name path string true "Snapshot to search"
+// @Param name path string true "Repository name"
 // @Param q query string true "Package query (e.g Name%20(~%20matlab))"
 // @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
 // @Param format query string true "Set to 'details' to return extra info about each package"
 // @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Internal Server Error"
@@ -348,10 +409,13 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
@@ -360,13 +424,23 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	resources := []string{string(repo.Key())}
 
 	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		out.Printf("Loading packages...\n")
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -375,7 +449,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		for _, ref := range b.PackageRefs {
 			var p *deb.Package
 
-			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			p, err = taskCollectionFactory.PackageCollection().ByKey([]byte(ref))
 			if err != nil {
 				if err == database.ErrNotFound {
 					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
@@ -391,7 +465,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -406,9 +480,11 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -426,9 +502,11 @@ func apiReposPackagesAdd(c *gin.Context) {
 // @Description
 // @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
 // @Tags Repos
-// @Produce json
-// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
+// @Consume  json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -449,7 +527,7 @@ func apiReposPackagesDelete(c *gin.Context) {
 // @Tags Repos
 // @Param name path string true "Repository name"
 // @Param dir path string true "Directory of packages"
-// @Param file path string false "Filename (optional)"
+// @Param file path string true "Filename"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 200 {string} string "OK"
@@ -494,6 +572,8 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
@@ -517,7 +597,17 @@ func apiReposPackageFromDir(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -538,13 +628,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err = deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+			taskCollectionFactory.PackageCollection(), reporter, nil, taskCollectionFactory.ChecksumCollection)
 		failedFiles = append(failedFiles, failedFiles2...)
 		processedFiles = append(processedFiles, otherFiles...)
 
@@ -554,7 +644,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -570,7 +660,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 			}
 
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -607,11 +697,11 @@ type reposCopyPackageParams struct {
 // @Summary Copy Package
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
-// @Produce json
-// @Param name path string true "Source repo"
-// @Param src path string true "Destination repo"
+// @Param name path string true "Destination repo"
+// @Param src path string true "Source repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -633,6 +723,8 @@ func apiReposCopyPackage(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource keys.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
 	if err != nil {
@@ -656,12 +748,26 @@ func apiReposCopyPackage(c *gin.Context) {
 	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
 
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		// Task: Create fresh factory and collections inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of both repos after lock acquired
+		dstRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(dstRepoName)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		srcRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(srcRepoName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
 		}
@@ -674,12 +780,12 @@ func apiReposCopyPackage(c *gin.Context) {
 			RemovedLines: []string{},
 		}
 
-		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
 		}
 
-		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
 		}
@@ -747,7 +853,7 @@ func apiReposCopyPackage(c *gin.Context) {
 		} else {
 			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			err = taskCollectionFactory.LocalRepoCollection().Update(dstRepo)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 			}
@@ -762,12 +868,15 @@ func apiReposCopyPackage(c *gin.Context) {
 // @Summary Include File from Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string true "File/packages to include"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir}/{file} [post]
@@ -776,26 +885,22 @@ func apiReposIncludePackageFromFile(c *gin.Context) {
 	apiReposIncludePackageFromDir(c)
 }
 
-type reposIncludePackageFromDirReport struct {
-	Warnings []string
-	Added    []string
-	Deleted  []string
-}
-
 type reposIncludePackageFromDirResponse struct {
-	Report      reposIncludePackageFromDirReport
+	Report      *aptly.RecordingResultReporter
 	FailedFiles []string
 }
 
 // @Summary Include Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} reposIncludePackageFromDirResponse "Response"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir} [post]
@@ -836,7 +941,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	}
 
 	var resources []string
-	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+	if len(repoTemplate.Root.Nodes) > 1 {
 		resources = append(resources, task.AllLocalReposResourcesKey)
 	} else {
 		// repo template string is simple text so only use resource key of specific repository
@@ -851,6 +956,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	resources = append(resources, sources...)
 
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()
@@ -866,8 +974,8 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
 		_, failedFiles2, err = deb.ImportChangesFiles(
 			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
-			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+			repoTemplate, context.Progress(), taskCollectionFactory.LocalRepoCollection(), taskCollectionFactory.PackageCollection(),
+			context.PackagePool(), taskCollectionFactory.ChecksumCollection, nil, query.Parse)
 		failedFiles = append(failedFiles, failedFiles2...)
 
 		if err != nil {
@@ -876,7 +984,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 		if !noRemoveFiles {
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -896,9 +1004,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
-			"Report":      reporter,
-			"FailedFiles": failedFiles,
-		}}, nil
+		ret := reposIncludePackageFromDirResponse{
+			Report:      reporter,
+			FailedFiles: failedFiles,
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
 	})
 }

api/router.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"net/http"
-	"os"
 	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -12,13 +11,19 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog/log"
 
-	"github.com/aptly-dev/aptly/docs"
-	swaggerFiles "github.com/swaggo/files"
-	ginSwagger "github.com/swaggo/gin-swagger"
+	// _ "github.com/aptly-dev/aptly/docs" // import docs
+	// swaggerFiles "github.com/swaggo/files"
+	// ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		countPackagesByRepos()
@@ -26,21 +31,21 @@ func apiMetricsGet() gin.HandlerFunc {
 	}
 }
 
-func redirectSwagger(c *gin.Context) {
-	if c.Request.URL.Path == "/docs/index.html" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs/" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	c.Next()
-}
+// func redirectSwagger(c *gin.Context) {
+// 	if c.Request.URL.Path == "/docs/index.html" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs/" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	c.Next()
+// }
 
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
@@ -56,26 +61,22 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	router.UseRawPath = true
 
 	if c.Config().LogFormat == "json" {
-		c.StructuredLogging(true)
-		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
 		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
 		router.Use(JSONLogger())
 	} else {
-		c.StructuredLogging(false)
-		utils.SetupDefaultLogger(c.Config().LogLevel)
 		router.Use(gin.Logger())
 	}
 
 	router.Use(gin.Recovery(), gin.ErrorLogger())
 
-	if c.Config().EnableSwaggerEndpoint {
-		router.GET("docs.html", func(c *gin.Context) {
-			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
-		})
-		router.Use(redirectSwagger)
-		url := ginSwagger.URL("/docs/doc.json")
-		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
-	}
+	// if c.Config().EnableSwaggerEndpoint {
+	// 	router.GET("docs.html", func(c *gin.Context) {
+	// 		c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+	// 	})
+	// 	router.Use(redirectSwagger)
+	// 	url := ginSwagger.URL("/docs/doc.json")
+	// 	router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	// }
 
 	if c.Config().EnableMetricsEndpoint {
 		MetricsCollectorRegistrar.Register(router)
@@ -220,7 +221,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.GET("/graph.:ext", apiGraph)
 	}
 	{
-		api.POST("/db/cleanup", apiDbCleanup)
+		api.POST("/db/cleanup", apiDBCleanup)
 	}
 	{
 		api.GET("/tasks", apiTasksList)

api/s3.go

@@ -14,7 +14,8 @@ import (
 // @Router /api/s3 [get]
 func apiS3List(c *gin.Context) {
 	keys := []string{}
-	for k := range context.Config().S3PublishRoots {
+	s3Roots := context.Config().S3PublishRoots
+	for k := range s3Roots {
 		keys = append(keys, k)
 	}
 	c.JSON(200, keys)

api/snapshot.go

@@ -33,7 +33,7 @@ func apiSnapshotsList(c *gin.Context) {
 	}
 
 	result := []*deb.Snapshot{}
-	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+	_ = collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
 		result = append(result, snapshot)
 		return nil
 	})
@@ -74,26 +74,33 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
 		}
 
-		err = collection.LoadComplete(repo)
+		err = taskMirrorCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -107,7 +114,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -156,6 +163,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		}
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 	var resources []string
@@ -169,37 +177,62 @@ func apiSnapshotsCreate(c *gin.Context) {
 			return
 		}
 
-		resources = append(resources, string(sources[i].ResourceKey()))
+		resources = append(resources, string(sources[i].Key()))
 	}
 
 	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		for i := range sources {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPackageCollection := taskCollectionFactory.PackageCollection()
+
+		// Fresh load of all sources after lock acquired
+		freshSources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+		for i := range b.SourceSnapshots {
+			freshSources[i], err = taskSnapshotCollection.ByName(b.SourceSnapshots[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
 		}
 
-		list := deb.NewPackageList()
+		// Merge packages from all source snapshots
+		var refList *deb.PackageRefList
+		if len(freshSources) > 0 {
+			refList = freshSources[0].RefList()
+			for i := 1; i < len(freshSources); i++ {
+				refList = refList.Merge(freshSources[i].RefList(), true, false)
+			}
+		} else {
+			refList = deb.NewPackageRefList()
+		}
 
-		// verify package refs and build package list
-		for _, ref := range b.PackageRefs {
-			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
-			if err != nil {
-				if err == database.ErrNotFound {
-					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+		// Add any explicitly specified package refs on top
+		if len(b.PackageRefs) > 0 {
+			list := deb.NewPackageList()
+			for _, ref := range b.PackageRefs {
+				p, err := taskPackageCollection.ByKey([]byte(ref))
+				if err != nil {
+					if err == database.ErrNotFound {
+						return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+					}
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+				}
+				err = list.Add(p)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 				}
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-			}
-			err = list.Add(p)
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
+			refList = refList.Merge(deb.NewPackageRefListFromPackageList(list), true, false)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+		snapshot = deb.NewSnapshotFromRefList(b.Name, freshSources, refList, b.Description)
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -217,10 +250,9 @@ type snapshotsCreateFromRepositoryParams struct {
 // @Summary Snapshot Repository
 // @Description **Create a snapshot of a repository by name**
 // @Tags Snapshots
-// @Param name path string true "Repository name"
 // @Consume json
 // @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
-// @Param name path string true "Name of the snapshot"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -241,21 +273,28 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.LoadComplete(repo)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskRepoCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskRepoCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskRepoCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -269,7 +308,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -307,6 +346,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
@@ -317,14 +357,38 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
-	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		_, err := collection.ByName(b.Name)
+	// Pre-task validation of new name if provided (skip if renaming to same name)
+	if b.Name != "" && b.Name != name {
+		_, err = collection.ByName(b.Name)
 		if err == nil {
-			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+			return
+		}
+	}
+
+	resources := []string{string(snapshot.Key())}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err = taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
+		// Fresh duplicate check inside lock
+		if b.Name != "" {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			}
+		}
+
+		// Update fresh copy
 		if b.Name != "" {
 			snapshot.Name = b.Name
 		}
@@ -333,7 +397,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		err = taskCollection.Update(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -387,9 +451,9 @@ func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
@@ -397,23 +461,37 @@ func apiSnapshotsDrop(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey())}
+	resources := []string{string(snapshot.Key())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.BySnapshot(snapshot)
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err := taskSnapshotCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Fresh checks with current collections
+		published := taskPublishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			// Using fresh collection for dependency check
+			snapshots := taskSnapshotCollection.BySnapshotSource(snapshot)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
 			}
 		}
 
-		err = snapshotCollection.Drop(snapshot)
+		err = taskSnapshotCollection.Drop(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -555,7 +633,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 	}
 
 	if len(body.Sources) < 1 {
-		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("minimum one source snapshot is required"))
 		return
 	}
 
@@ -568,6 +646,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 
@@ -580,36 +659,47 @@ func apiSnapshotsMerge(c *gin.Context) {
 			return
 		}
 
-		resources[i] = string(sources[i].ResourceKey())
+		resources[i] = string(sources[i].Key())
 	}
 
 	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = snapshotCollection.LoadComplete(sources[0])
-		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-		}
-		result := sources[0].RefList()
-		for i := 1; i < len(sources); i++ {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load of all sources inside task
+		freshSources := make([]*deb.Snapshot, len(body.Sources))
+		for i := range body.Sources {
+			freshSources[i], err = taskSnapshotCollection.ByName(body.Sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
-			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+		}
+
+		// Merge using fresh sources
+		result := freshSources[0].RefList()
+		for i := 1; i < len(freshSources); i++ {
+			result = result.Merge(freshSources[i].RefList(), overrideMatching, false)
 		}
 
 		if latest {
 			result.FilterLatestRefs()
 		}
 
-		sourceDescription := make([]string, len(sources))
-		for i, s := range sources {
+		sourceDescription := make([]string, len(freshSources))
+		for i, s := range freshSources {
 			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+		snapshot = deb.NewSnapshotFromRefList(name, freshSources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -690,24 +780,32 @@ func apiSnapshotsPull(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	resources := []string{string(sourceSnapshot.Key()), string(toSnapshot.Key())}
 	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of snapshots after lock acquired
+		freshToSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(name)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+		freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshSourceSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		// convert snapshots to package list
-		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		toPackageList, err := deb.NewPackageListFromRefList(freshToSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		sourcePackageList, err := deb.NewPackageListFromRefList(freshSourceSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -765,7 +863,7 @@ func apiSnapshotsPull(c *gin.Context) {
 		addedPackages := []string{}
 		alreadySeen := map[string]bool{}
 
-		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+		_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
 			key := pkg.Architecture + "_" + pkg.Name
 			_, seen := alreadySeen[key]
 
@@ -781,7 +879,7 @@ func apiSnapshotsPull(c *gin.Context) {
 
 			// If !allMatches, add only first matching name-arch package
 			if !seen || allMatches {
-				toPackageList.Add(pkg)
+				_ = toPackageList.Add(pkg)
 				addedPackages = append(addedPackages, pkg.String())
 			}
 
@@ -804,10 +902,10 @@ func apiSnapshotsPull(c *gin.Context) {
 		}
 
 		// Create <destination> snapshot
-		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
-			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{freshToSnapshot, freshSourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", freshToSnapshot.Name, freshSourceSnapshot.Name, strings.Join(body.Queries, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(destinationSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}

aptly/conf.go

@@ -1,4 +1,4 @@
 package aptly
 
-// Default aptly.conf (filled in at link time)
+// AptlyConf holds the default aptly.conf (filled in at link time)
 var AptlyConf []byte

aptly/interfaces.go

@@ -95,8 +95,8 @@ type FileSystemPublishedStorage interface {
 
 // PublishedStorageProvider is a thing that returns PublishedStorage by name
 type PublishedStorageProvider interface {
-	// GetPublishedStorage returns PublishedStorage by name
-	GetPublishedStorage(name string) PublishedStorage
+	// GetPublishedStorage returns PublishedStorage by name, or an error if the storage is not configured
+	GetPublishedStorage(name string) (PublishedStorage, error)
 }
 
 // BarType used to differentiate between different progress bars

azure/azure.go

@@ -5,35 +5,28 @@ package azure
 import (
 	"context"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net/url"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 )
 
 func isBlobNotFound(err error) bool {
-	var respErr *azcore.ResponseError
-	if errors.As(err, &respErr) {
-		return respErr.StatusCode == 404 // BlobNotFound
-	}
-	return false
+	storageError, ok := err.(azblob.StorageError)
+	return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
 }
 
 type azContext struct {
-	client    *azblob.Client
-	container string
+	container azblob.ContainerURL
 	prefix    string
 }
 
 func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
-	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
 	if err != nil {
 		return nil, err
 	}
@@ -42,14 +35,15 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) (
 		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
 	}
 
-	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
 	if err != nil {
 		return nil, err
 	}
 
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
 	result := &azContext{
-		client:    serviceClient,
-		container: container,
+		container: containerURL,
 		prefix:    prefix,
 	}
 
@@ -60,6 +54,10 @@ func (az *azContext) blobPath(path string) string {
 	return filepath.Join(az.prefix, path)
 }
 
+func (az *azContext) blobURL(path string) azblob.BlobURL {
+	return az.container.NewBlobURL(az.blobPath(path))
+}
+
 func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
 	const delimiter = "/"
 	paths = make([]string, 0, 1024)
@@ -69,33 +67,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 		prefix += delimiter
 	}
 
-	ctx := context.Background()
-	maxResults := int32(1)
-	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
-		Prefix:     &prefix,
-		MaxResults: &maxResults,
-		Include:    azblob.ListBlobsInclude{Metadata: true},
-	})
-
-	// Iterate over each page
-	for pager.More() {
-		page, err := pager.NextPage(ctx)
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := az.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
 		if err != nil {
 			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
 		}
 
-		for _, blob := range page.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, *blob.Name)
-			} else {
-				name := *blob.Name
-				paths = append(paths, name[len(prefix):])
-			}
-			b := *blob
-			md5 := b.Properties.ContentMD5
-			md5s = append(md5s, fmt.Sprintf("%x", md5))
+		marker = listBlob.NextMarker
 
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
 		}
+
 		if progress != nil {
 			time.Sleep(time.Duration(500) * time.Millisecond)
 			progress.AddBar(1)
@@ -105,27 +97,28 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 	return paths, md5s, nil
 }
 
-func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
-	uploadOptions := &azblob.UploadFileOptions{
-		BlockSize:   4 * 1024 * 1024,
-		Concurrency: 8,
+func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
 	}
 
-	path := az.blobPath(blobName)
 	if len(sourceMD5) > 0 {
 		decodedMD5, err := hex.DecodeString(sourceMD5)
 		if err != nil {
 			return err
 		}
-		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
-			BlobContentMD5: decodedMD5,
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
 		}
 	}
 
-	var err error
-	if file, ok := source.(*os.File); ok {
-		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
-	}
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob.ToBlockBlobURL(),
+		uploadOptions,
+	)
 
 	return err
 }

azure/package_pool.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/pkg/errors"
@@ -29,7 +30,7 @@ func NewPackagePool(accountName, accountKey, container, prefix, endpoint string)
 	return &PackagePool{az: azctx}, nil
 }
 
-// String
+// String returns the storage as string
 func (pool *PackagePool) String() string {
 	return pool.az.String()
 }
@@ -40,7 +41,10 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu
 	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
 }
 
-func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+func (pool *PackagePool) ensureChecksums(
+	poolPath string,
+	checksumStorage aptly.ChecksumStorage,
+) (*utils.ChecksumInfo, error) {
 	targetChecksums, err := checksumStorage.Get(poolPath)
 	if err != nil {
 		return nil, err
@@ -48,7 +52,8 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 
 	if targetChecksums == nil {
 		// we don't have checksums stored yet for this file
-		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		blob := pool.az.blobURL(poolPath)
+		download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 		if err != nil {
 			if isBlobNotFound(err) {
 				return nil, nil
@@ -58,7 +63,7 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 		}
 
 		targetChecksums = &utils.ChecksumInfo{}
-		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
 		if err != nil {
 			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
 		}
@@ -87,49 +92,45 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er
 }
 
 func (pool *PackagePool) Size(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	blob := pool.az.blobURL(path)
+
 	temp, err := os.CreateTemp("", "blob-download")
 	if err != nil {
-		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+		return nil, errors.Wrap(err, "error creating temporary file for blob download")
 	}
-	defer os.Remove(temp.Name())
+	defer func() { _ = os.Remove(temp.Name()) }()
 
-	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
 	if err != nil {
-		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+		return nil, errors.Wrapf(err, "error downloading blob at %s", path)
 	}
 
 	return temp, nil
 }
 
 func (pool *PackagePool) Remove(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+		return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
 	}
 
-	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
@@ -143,6 +144,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	}
 
 	path := pool.buildPoolPath(basename, checksums)
+	blob := pool.az.blobURL(path)
 	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
 	if err != nil {
 		return "", err
@@ -156,9 +158,9 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	if err != nil {
 		return "", err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
-	err = pool.az.putFile(path, source, checksums.MD5)
+	err = pool.az.putFile(blob, source, checksums.MD5)
 	if err != nil {
 		return "", err
 	}

azure/package_pool_test.go

@@ -2,12 +2,12 @@ package azure
 
 import (
 	"context"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
@@ -50,10 +50,8 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
 
 	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
-        })
+	cnt := s.pool.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -69,8 +67,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []string{})
 
-	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
-	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
 
 	list, err = s.pool.FilepathList(nil)
 	c.Check(err, IsNil)
@@ -81,8 +79,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
 }
 
 func (s *PackagePoolSuite) TestRemove(c *C) {
-	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
-	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
 
 	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
 	c.Check(err, IsNil)
@@ -247,7 +245,7 @@ func (s *PackagePoolSuite) TestOpen(c *C) {
 
 	f, err := s.pool.Open(path)
 	c.Assert(err, IsNil)
-	contents, err := ioutil.ReadAll(f)
+	contents, err := io.ReadAll(f)
 	c.Assert(err, IsNil)
 	c.Check(len(contents), Equals, 2738)
 	c.Check(f.Close(), IsNil)

azure/public.go

@@ -3,22 +3,19 @@ package azure
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/pborman/uuid"
 	"github.com/pkg/errors"
 )
 
 // PublishedStorage abstract file system with published files (actually hosted on Azure)
 type PublishedStorage struct {
-	prefix    string
 	az        *azContext
 	pathCache map[string]map[string]string
 }
@@ -38,7 +35,7 @@ func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint st
 	return &PublishedStorage{az: azctx}, nil
 }
 
-// String
+// String returns the storage as string
 func (storage *PublishedStorage) String() string {
 	return storage.az.String()
 }
@@ -65,9 +62,9 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	if err != nil {
 		return err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -77,15 +74,14 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 
 // RemoveDirs removes directory structure under public path
 func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
-	path = storage.az.blobPath(path)
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := filepath.Join(path, filename)
-		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
+		blob := storage.az.blobURL(filepath.Join(path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
 		}
@@ -96,8 +92,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	path = storage.az.blobPath(path)
-	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
+	blob := storage.az.blobURL(path)
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
 	}
@@ -116,8 +112,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
 	relFilePath := filepath.Join(publishedRelPath, fileName)
-	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
-	poolPath := storage.az.blobPath(prefixRelFilePath)
+	// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	// FIXME: check how to integrate publishedPrefix:
+	poolPath := storage.az.blobPath(fileName)
 
 	if storage.pathCache == nil {
 		storage.pathCache = make(map[string]map[string]string)
@@ -158,9 +155,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	if err != nil {
 		return err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(relFilePath, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
 	if err == nil {
 		pathCache[relFilePath] = sourceMD5
 	} else {
@@ -177,58 +174,57 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 }
 
 // Internal copy or move implementation
-func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
 	const leaseDuration = 30
-	leaseID := uuid.NewRandom().String()
 
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	srcBlobClient := containerClient.NewBlobClient(src)
-	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	dstBlobURL := storage.az.blobURL(dst)
+	srcBlobURL := storage.az.blobURL(src)
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
 	}
+	defer func() { _, _ = srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) }()
+	srcBlobLeaseID := leaseResp.LeaseID()
 
-	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
-	}
-	defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
-
-	dstBlobClient := containerClient.NewBlobClient(dst)
-	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
-		Metadata: metadata,
-	})
-
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
 	}
 
-	copyStatus := *copyResp.CopyStatus
+	copyStatus := copyResp.CopyStatus()
 	for {
-		if copyStatus == blob.CopyStatusTypeSuccess {
+		if copyStatus == azblob.CopyStatusSuccess {
 			if move {
-				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
-					AccessConditions: &blob.AccessConditions{
-						LeaseAccessConditions: &blob.LeaseAccessConditions{
-							LeaseID: &leaseID,
-						},
-					},
-				})
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
 				return err
 			}
 			return nil
-		} else if copyStatus == blob.CopyStatusTypePending {
+		} else if copyStatus == azblob.CopyStatusPending {
 			time.Sleep(1 * time.Second)
-			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
 			if err != nil {
-				return fmt.Errorf("error getting copy progress %s", dst)
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
 			}
-			copyStatus = *getMetadata.CopyStatus
+			copyStatus = blobPropsResp.CopyStatus()
 
-			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
 			if err != nil {
-				return fmt.Errorf("error renewing source blob lease %s", src)
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
 			}
 		} else {
 			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -243,9 +239,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 // SymLink creates a copy of src file and adds link information as meta data
 func (storage *PublishedStorage) SymLink(src string, dst string) error {
-	metadata := make(map[string]*string)
-	metadata["SymLink"] = &src
-	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
 }
 
 // HardLink using symlink functionality as hard links do not exist
@@ -255,33 +249,28 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	_, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		if isBlobNotFound(err) {
 			return false, nil
 		}
-		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
 	}
-	return true, nil
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 }
 
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	props, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return "", fmt.Errorf("failed to get blob properties: %v", err)
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 	}
-
-	metadata := props.Metadata
-	if originalBlob, exists := metadata["original_blob"]; exists {
-		return *originalBlob, nil
-	}
-	return "", fmt.Errorf("error reading link %s: %v", path, err)
+	return resp.NewMetadata()["SymLink"], nil
 }

azure/public_test.go

@@ -4,14 +4,11 @@ import (
 	"context"
 	"crypto/md5"
 	"crypto/rand"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
-        "bytes"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 	. "gopkg.in/check.v1"
@@ -36,7 +33,7 @@ func randString(n int) string {
 	}
 	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
 	var bytes = make([]byte, n)
-	rand.Read(bytes)
+	_, _ = rand.Read(bytes)
 	for i, b := range bytes {
 		bytes[i] = alphanum[b%byte(len(alphanum))]
 	}
@@ -69,10 +66,8 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
-        })
+	cnt := s.storage.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -80,39 +75,41 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	cnt := s.storage.az.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	blob := s.storage.az.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, IsNil)
-	data, err := ioutil.ReadAll(resp.Body)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := io.ReadAll(body)
 	c.Assert(err, IsNil)
 	return data
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-        serviceClient := s.storage.az.client.ServiceClient()
-        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
-        blobClient := containerClient.NewBlobClient(path)
-        _, err := blobClient.GetProperties(context.Background(), nil)
+	_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, NotNil)
-
-        storageError, ok := err.(*azcore.ResponseError)
+	storageError, ok := err.(azblob.StorageError)
 	c.Assert(ok, Equals, true)
-	c.Assert(storageError.StatusCode, Equals, 404)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
 }
 
 func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	hash := md5.Sum(data)
-        uploadOptions := &azblob.UploadStreamOptions{
-            HTTPHeaders: &blob.HTTPHeaders{
-            	BlobContentMD5: hash[:],
-            },
-        }
-        reader := bytes.NewReader(data)
-        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.az.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
 	c.Assert(err, IsNil)
 }
 
@@ -121,7 +118,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	filename := "a/b.txt"
 
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -140,7 +137,7 @@ func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
 	filename := "a/b+c.txt"
 
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -258,7 +255,7 @@ func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
@@ -280,18 +277,18 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	cs := files.NewMockChecksumStorage()
 
 	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
-	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	err := os.WriteFile(tmpFile1, []byte("Contents"), 0644)
 	c.Assert(err, IsNil)
 	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
 
 	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
-	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	err = os.WriteFile(tmpFile2, []byte("Spam"), 0644)
 	c.Assert(err, IsNil)
 	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
 
 	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
-	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
-	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	_ = os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	err = os.WriteFile(tmpFile3, []byte("Contents"), 0644)
 	c.Assert(err, IsNil)
 	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
 
@@ -333,7 +330,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 
 	// 2nd link from pool, providing wrong path for source file
 	//
-	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
 	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)

cmd/api_serve.go

@@ -46,7 +46,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	}
 	if err == nil && len(listeners) == 1 {
 		listener := listeners[0]
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
 		err = http.Serve(listener, api.Router(context))
 		if err != nil {
@@ -67,7 +67,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		if _, ok := <-sigchan; ok {
 			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
 			context.TaskList().Wait()
-			server.Shutdown(stdcontext.Background())
+			_ = server.Shutdown(stdcontext.Background())
 		}
 	})()
 	defer close(sigchan)
@@ -75,14 +75,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-		os.Remove(file)
+		_ = os.Remove(file)
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)
 		if err != nil {
 			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
 		}
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 
 		err = server.Serve(listener)
 	} else {

cmd/cmd.go

@@ -97,7 +97,7 @@ package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
 			makeCmdConfig(),
-			makeCmdDb(),
+			makeCmdDB(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),

cmd/config_show.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/smira/commander"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func aptlyConfigShow(_ *commander.Command, _ []string) error {

cmd/db.go

@@ -4,13 +4,13 @@ import (
 	"github.com/smira/commander"
 )
 
-func makeCmdDb() *commander.Command {
+func makeCmdDB() *commander.Command {
 	return &commander.Command{
 		UsageLine: "db",
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
-			makeCmdDbCleanup(),
-			makeCmdDbRecover(),
+			makeCmdDBCleanup(),
+			makeCmdDBRecover(),
 		},
 	}
 }

cmd/db_cleanup.go

@@ -12,7 +12,7 @@ import (
 )
 
 // aptly db cleanup
-func aptlyDbCleanup(cmd *commander.Command, args []string) error {
+func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -48,7 +48,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 			if verbose {
 				description := fmt.Sprintf("mirror %s", repo.Name)
-				repo.RefList().ForEach(func(key []byte) error {
+				_ = repo.RefList().ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -81,7 +81,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 			if verbose {
 				description := fmt.Sprintf("local repo %s", repo.Name)
-				repo.RefList().ForEach(func(key []byte) error {
+				_ = repo.RefList().ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -113,7 +113,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 		if verbose {
 			description := fmt.Sprintf("snapshot %s", snapshot.Name)
-			snapshot.RefList().ForEach(func(key []byte) error {
+			_ = snapshot.RefList().ForEach(func(key []byte) error {
 				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 				return nil
 			})
@@ -146,7 +146,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			if verbose {
 				description := fmt.Sprintf("published repository %s:%s/%s component %s",
 					published.Storage, published.Prefix, published.Distribution, component)
-				published.RefList(component).ForEach(func(key []byte) error {
+				_ = published.RefList(component).ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -291,9 +291,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func makeCmdDbCleanup() *commander.Command {
+func makeCmdDBCleanup() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbCleanup,
+		Run:       aptlyDBCleanup,
 		UsageLine: "cleanup",
 		Short:     "cleanup DB and package pool",
 		Long: `

cmd/db_recover.go

@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 
 	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
-func aptlyDbRecover(cmd *commander.Command, args []string) error {
+func aptlyDBRecover(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -16,14 +19,19 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = goleveldb.RecoverDB(context.DBPath())
+	if err = goleveldb.RecoverDB(context.DBPath()); err != nil {
+		return err
+	}
+
+	context.Progress().Printf("Checking database integrity...\n")
+	err = checkIntegrity()
 
 	return err
 }
 
-func makeCmdDbRecover() *commander.Command {
+func makeCmdDBRecover() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbRecover,
+		Run:       aptlyDBRecover,
 		UsageLine: "recover",
 		Short:     "recover DB after crash",
 		Long: `
@@ -38,3 +46,36 @@ Example:
 
 	return cmd
 }
+
+func checkIntegrity() error {
+	return context.NewCollectionFactory().LocalRepoCollection().ForEach(checkRepo)
+}
+
+func checkRepo(repo *deb.LocalRepo) error {
+	collectionFactory := context.NewCollectionFactory()
+	repos := collectionFactory.LocalRepoCollection()
+
+	err := repos.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("load complete repo %q: %s", repo.Name, err)
+	}
+
+	dangling, err := deb.FindDanglingReferences(repo.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		return fmt.Errorf("find dangling references: %w", err)
+	}
+
+	if len(dangling.Refs) > 0 {
+		for _, ref := range dangling.Refs {
+			context.Progress().Printf("Removing dangling database reference %q\n", ref)
+		}
+
+		repo.UpdateRefList(repo.RefList().Subtract(dangling))
+
+		if err = repos.Update(repo); err != nil {
+			return fmt.Errorf("update repo: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/graph.go

@@ -38,8 +38,8 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	tempfile.Close()
-	os.Remove(tempfile.Name())
+	_ = tempfile.Close()
+	_ = os.Remove(tempfile.Name())
 
 	format := context.Flags().Lookup("format").Value.String()
 	output := context.Flags().Lookup("output").Value.String()

cmd/mirror.go

@@ -20,7 +20,7 @@ func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
+	err := verifier.InitKeyring(!ignoreSignatures) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_list.go

@@ -32,7 +32,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 
 	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -42,7 +42,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -70,13 +70,13 @@ func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
 
 	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		repos[i] = repo
 		i++
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/mirror_show.go

@@ -86,7 +86,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList(), collectionFactory)
+			_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
@@ -119,7 +119,7 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				repo.Packages = append(repo.Packages, p.GetFullName())
 				return nil
 			})

cmd/mirror_update.go

@@ -101,7 +101,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			collectionFactory.RemoteRepoCollection().Update(repo)
+			_ = collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
@@ -173,7 +173,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						file, e = os.CreateTemp("", task.File.Filename)
 						if e == nil {
 							task.TempDownPath = file.Name()
-							file.Close()
+							_ = file.Close()
 						}
 					}
 					if e != nil {
@@ -261,7 +261,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
-	repo.FinalizeDownload(collectionFactory, context.Progress())
+	_ = repo.FinalizeDownload(collectionFactory, context.Progress())
 	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)

cmd/package_search.go

@@ -40,7 +40,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }

cmd/package_show.go

@@ -84,8 +84,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false, false)
-		w.Flush()
+		_ = p.Stanza().WriteTo(w, p.IsSource, false, false)
+		_ = w.Flush()
 		fmt.Printf("\n")
 
 		if withFiles {
@@ -109,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p, collectionFactory)
+			_ = printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 

cmd/publish.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+
+	var gpgKeys []string
+
+	// CLI args have priority over config
+	cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
+	if len(cliKeys) > 0 {
+		gpgKeys = cliKeys
+	} else if len(context.Config().GpgKeys) > 0 {
+		gpgKeys = context.Config().GpgKeys
+	}
+
+	for _, gpgKey := range gpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 
 }
 
+type gpgKeyFlag struct {
+	gpgKeys []string
+}
+
+func (k *gpgKeyFlag) Set(value string) error {
+	k.gpgKeys = append(k.gpgKeys, value)
+	return nil
+}
+
+func (k *gpgKeyFlag) Get() interface{} {
+	return k.gpgKeys
+}
+
+func (k *gpgKeyFlag) String() string {
+	return strings.Join(k.gpgKeys, ",")
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_list.go

@@ -53,7 +53,7 @@ func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(published)
 
@@ -99,7 +99,7 @@ func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].GetPath() < repos[j].GetPath()

cmd/publish_repo.go

@@ -34,7 +34,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_snapshot.go

@@ -156,7 +156,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+		_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -190,9 +190,11 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
-		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
-			localStorage.PublicPath())
+	if ps, err := context.GetPublishedStorage(storage); err == nil {
+		if localStorage, ok := ps.(aptly.FileSystemPublishedStorage); ok {
+			context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+				localStorage.PublicPath())
+		}
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
@@ -230,7 +232,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_switch.go

@@ -151,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_update.go

@@ -115,7 +115,7 @@ Example:
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/repo_list.go

@@ -32,7 +32,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -47,7 +47,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -76,7 +76,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
@@ -87,7 +87,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/repo_remove.go

@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *deb.Package) error {
+	_ = toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
 		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil

cmd/repo_show.go

@@ -52,7 +52,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList(), collectionFactory)
+		_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 	}
 
 	return err

cmd/serve.go

@@ -97,7 +97,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage("")
+	if err != nil {
+		return err
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/snapshot_list.go

@@ -33,7 +33,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
 	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
-		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
 			return nil
 		})
@@ -68,7 +68,7 @@ func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
 
 	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
 	i := 0
-	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+	_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 		jsonSnapshots[i] = snapshot
 		i++
 		return nil

cmd/snapshot_pull.go

@@ -116,7 +116,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	alreadySeen := map[string]bool{}
 
-	result.ForEachIndexed(func(pkg *deb.Package) error {
+	_ = result.ForEachIndexed(func(pkg *deb.Package) error {
 		key := pkg.Architecture + "_" + pkg.Name
 		_, seen := alreadySeen[key]
 
@@ -132,7 +132,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 		// If !allMatches, add only first matching name-arch package
 		if !seen || allMatches {
-			packageList.Add(pkg)
+			_ = packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
 		}
 

cmd/snapshot_search.go

@@ -123,7 +123,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }

cmd/snapshot_show.go

@@ -79,7 +79,7 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+		_ = ListPackagesRefList(snapshot.RefList(), collectionFactory)
 	}
 
 	return err
@@ -139,7 +139,7 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
 				return nil
 			})

cmd/task_run.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mattn/go-shellwords"
+	shellwords "github.com/mattn/go-shellwords"
 	"github.com/smira/commander"
 )
 
@@ -31,7 +31,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		defer file.Close()
+		defer func() { _ = file.Close() }()
 
 		scanner := bufio.NewScanner(file)
 

completion.d/aptly

@@ -22,34 +22,36 @@
 
 __aptly_mirror_list()
 {
-  aptly mirror list -raw
+  aptly ${aptly_global_opts[@]} mirror list -raw
 }
 
 __aptly_repo_list()
 {
-  aptly repo list -raw
+  aptly ${aptly_global_opts[@]} repo list -raw
 }
 
 __aptly_snapshot_list()
 {
-  aptly snapshot list -raw
+  aptly ${aptly_global_opts[@]} snapshot list -raw
 }
 
 __aptly_published_distributions()
 {
-  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 2 | sort | uniq
 }
 
 __aptly_published_prefixes()
 {
-  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 1 | sort | uniq
 }
 
 __aptly_prefixes_for_distribution()
 {
-  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
 }
 
+
+
 _aptly()
 {
     cur="${COMP_WORDS[COMP_CWORD]}"
@@ -57,7 +59,12 @@ _aptly()
     prevprev="${COMP_WORDS[COMP_CWORD-2]}"
 
     commands="api config db graph mirror package publish repo serve snapshot task version"
-    options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
+
+    options="-architectures -config -db-open-attempts -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider"
+    options_without_arg="-dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve"
+    options_with_arg="-architectures -db-open-attempts -gpg-provider"
+    options_with_path_arg="-config"
+
     db_subcommands="cleanup recover"
     mirror_subcommands="create drop edit show list rename search update"
     publish_subcommands="drop list repo snapshot switch update source"
@@ -69,12 +76,41 @@ _aptly()
     config_subcommands="show"
     api_subcommands="serve"
 
-    local cmd subcmd numargs numoptions i
+    local cmd subcmd numargs numoptions i aptly_global_opts
 
     numargs=0
     numoptions=0
 
+    for opt in "${options_with_path_arg[@]}"; do
+        [[ "$prev" == "$opt" ]] || continue
+        compopt -o filenames 2>/dev/null
+        _filedir
+        return 0
+    done
+
     for (( i=1; i < $COMP_CWORD; i++ )); do
+      word=${COMP_WORDS[i]}
+      if [[ "$word" == -*=* ]]; then
+        for o in "${options[@]}"; do
+          [[ ${word%%=*} == "$o" ]] && aptly_global_opts+=("$word")
+        done
+      else
+        for o in "${options_with_arg[@]}" ""${options_with_path_arg[@]}"" ; do
+          if [[ "$word" == "$o" ]]; then
+            if (( i + 1 < COMP_CWORD )); then
+                aptly_global_opts+=("$word" "${COMP_WORDS[i+1]}")
+            else
+                aptly_global_opts+=("$word")
+            fi
+            (( i++ ))
+            continue 2
+          fi
+        done
+      fi
+      for o in ${options_without_arg[@]}; do
+          [[ "$word" == "$o" ]] && aptly_global_opts+=("$word")
+      done
+
       if [[ -n "$cmd" ]]; then
         if [[ ! -n "$subcmd" ]]; then
           subcmd=${COMP_WORDS[i]}
@@ -339,7 +375,7 @@ _aptly()
                 if [[ "$cur" == -* ]]; then
                   COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
                 else
-                  comptopt -o filenames 2>/dev/null
+                  compopt -o filenames 2>/dev/null
                   COMPREPLY=($(compgen -f -- ${cur}))
                   return 0
                 fi

console/progress.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/cheggaaa/pb"
 	"github.com/rs/zerolog/log"
 	"github.com/wsxiaoys/terminal/color"
@@ -78,7 +79,7 @@ func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}
-	if RunningOnTerminal() {
+	if utils.RunningOnTerminal() {
 		p.bar = pb.New(0)
 		p.bar.Total = count
 		p.bar.NotPrint = true
@@ -141,7 +142,7 @@ func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
 
 // ColoredPrintf does printf in colored way + newline
 func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
-	if RunningOnTerminal() {
+	if utils.RunningOnTerminal() {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks

console/progress_test.go

@@ -11,7 +11,7 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ProgressSuite struct {}
+type ProgressSuite struct{}
 
 var _ = Suite(&ProgressSuite{})
 

console/terminal.go

@@ -1,12 +0,0 @@
-package console
-
-import (
-	"syscall"
-
-	"golang.org/x/term"
-)
-
-// RunningOnTerminal checks whether stdout is terminal
-func RunningOnTerminal() bool {
-	return term.IsTerminal(syscall.Stdout)
-}

context/context.go

@@ -100,6 +100,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
 
 			for _, configLocation := range configLocations {
+				// FIXME: check if exists, check if readable
 				err = utils.LoadConfig(configLocation, &utils.Config)
 				if os.IsPermission(err) || os.IsNotExist(err) {
 					continue
@@ -115,7 +116,12 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
-				utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
+				defaultConfig := aptly.AptlyConf
+				if len(defaultConfig) == 0 {
+					defaultConfig = []byte("root_dir: \"\"")
+				}
+
+				_ = utils.SaveConfigRaw(homeLocation, defaultConfig)
 				err = utils.LoadConfig(homeLocation, &utils.Config)
 				if err != nil {
 					Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -123,6 +129,14 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 		}
 
+		if utils.Config.LogFormat == "json" {
+			context.StructuredLogging(true)
+			utils.SetupJSONLogger(utils.Config.LogLevel, os.Stdout)
+		} else {
+			context.StructuredLogging(false)
+			utils.SetupDefaultLogger(utils.Config.LogLevel)
+		}
+
 		context.configLoaded = true
 
 	}
@@ -233,7 +247,7 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo
 		// If flag is defined prefer it to global setting
 		maxTries = maxTriesFlag.Value.Get().(int)
 	}
-	var downloader string = context.config().Downloader
+	var downloader = context.config().Downloader
 	downloaderFlag := context.flags.Lookup("downloader")
 	if downloaderFlag != nil {
 		downloader = downloaderFlag.Value.String()
@@ -295,8 +309,8 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 		switch context.config().DatabaseBackend.Type {
 		case "leveldb":
 			dbPath := filepath.Join(context.config().GetRootDir(), "db")
-			if len(context.config().DatabaseBackend.DbPath) != 0 {
-				dbPath = context.config().DatabaseBackend.DbPath
+			if len(context.config().DatabaseBackend.DBPath) != 0 {
+				dbPath = context.config().DatabaseBackend.DBPath
 			}
 			context.database, err = goleveldb.NewDB(dbPath)
 		case "etcd":
@@ -398,8 +412,8 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	return context.packagePool
 }
 
-// GetPublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
+// GetPublishedStorage returns instance of PublishedStorage, or an error if the storage is not configured
+func (context *AptlyContext) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	context.Lock()
 	defer context.Unlock()
 
@@ -410,14 +424,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 		} else if strings.HasPrefix(name, "filesystem:") {
 			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
-				Fatal(fmt.Errorf("published local storage %v not configured", name[11:]))
+				return nil, fmt.Errorf("published local storage %v not configured", name[11:])
 			}
 
 			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
 		} else if strings.HasPrefix(name, "s3:") {
 			params, ok := context.config().S3PublishRoots[name[3:]]
 			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+				return nil, fmt.Errorf("published S3 storage %v not configured", name[3:])
 			}
 
 			var err error
@@ -427,39 +441,39 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
 				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "swift:") {
 			params, ok := context.config().SwiftPublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Swift storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
 				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "azure:") {
 			params, ok := context.config().AzurePublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Azure storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = azure.NewPublishedStorage(
 				params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+			return nil, fmt.Errorf("unknown published storage format: %v", name)
 		}
 		context.publishedStorages[name] = publishedStorage
 	}
 
-	return publishedStorage
+	return publishedStorage, nil
 }
 
 // UploadPath builds path to upload storage
@@ -589,17 +603,17 @@ func (context *AptlyContext) Shutdown() {
 
 	if aptly.EnableDebug {
 		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
+			_ = pprof.WriteHeapProfile(context.fileMemProfile)
+			_ = context.fileMemProfile.Close()
 			context.fileMemProfile = nil
 		}
 		if context.fileCPUProfile != nil {
 			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
+			_ = context.fileCPUProfile.Close()
 			context.fileCPUProfile = nil
 		}
 		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
+			_ = context.fileMemProfile.Close()
 			context.fileMemProfile = nil
 		}
 	}
@@ -607,7 +621,7 @@ func (context *AptlyContext) Shutdown() {
 		context.taskList.Stop()
 	}
 	if context.database != nil {
-		context.database.Close()
+		_ = context.database.Close()
 		context.database = nil
 	}
 	if context.downloader != nil {
@@ -652,7 +666,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 			if err != nil {
 				return nil, err
 			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
+			_ = pprof.StartCPUProfile(context.fileCPUProfile)
 		}
 
 		memprofile := flags.Lookup("memprofile").Value.String()
@@ -672,7 +686,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 				return nil, err
 			}
 
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+			_, _ = context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
 
 			go func() {
 				var stats runtime.MemStats
@@ -682,7 +696,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 				for {
 					runtime.ReadMemStats(&stats)
 					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+						_, _ = context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
 							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
 						time.Sleep(interval)
 					} else {

context/context_test.go

@@ -1,8 +1,6 @@
 package context
 
 import (
-	"fmt"
-	"os"
 	"reflect"
 	"testing"
 
@@ -80,10 +78,9 @@ func (s *AptlyContextSuite) SetUpTest(c *C) {
 
 func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
 	// https://github.com/aptly-dev/aptly/issues/711
-	// This will fail on account of us not having a config, so the
-	// storage never exists.
-	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
-		FatalErrorPanicMatches,
-		&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)",
-			os.Getenv("HOME"))})
+	// https://github.com/aptly-dev/aptly/issues/1477
+	// GetPublishedStorage must return an error (not panic) when the
+	// requested storage is not configured.
+	_, err := s.context.GetPublishedStorage("filesystem:fuji")
+	c.Assert(err, NotNil)
 }

database/etcddb/database_test.go

@@ -14,8 +14,7 @@ func Test(t *testing.T) {
 }
 
 type EtcDDBSuite struct {
-	url string
-	db  database.Storage
+	db database.Storage
 }
 
 var _ = Suite(&EtcDDBSuite{})
@@ -67,17 +66,17 @@ func (s *EtcDDBSuite) TestDelete(c *C) {
 func (s *EtcDDBSuite) TestByPrefix(c *C) {
 	//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
@@ -109,7 +108,7 @@ func (s *EtcDDBSuite) TestHasPrefix(c *C) {
 	//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
 	//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
 
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -124,15 +123,17 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 		value2 = []byte("value2")
 	)
 	transaction, err := s.db.OpenTransaction()
+	c.Assert(err, IsNil)
 
 	err = s.db.Put(key, value)
 	c.Assert(err, IsNil)
 
 	c.Assert(err, IsNil)
-	transaction.Put(key2, value2)
+	_ = transaction.Put(key2, value2)
 	v, err := s.db.Get(key)
+	c.Assert(err, IsNil)
 	c.Check(v, DeepEquals, value)
-        err = transaction.Delete(key)
+	err = transaction.Delete(key)
 	c.Assert(err, IsNil)
 
 	_, err = transaction.Get(key2)
@@ -155,4 +156,3 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 	_, err = transaction.Get(key)
 	c.Assert(err, NotNil)
 }
-

database/etcddb/storage.go

@@ -1,11 +1,11 @@
 package etcddb
 
 import (
-	"github.com/aptly-dev/aptly/database"
-	"github.com/pborman/uuid"
-	clientv3 "go.etcd.io/etcd/client/v3"
-
 	"fmt"
+
+	"github.com/aptly-dev/aptly/database"
+	"github.com/google/uuid"
+	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
 type EtcDStorage struct {
@@ -16,7 +16,7 @@ type EtcDStorage struct {
 
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
-	tmp := uuid.NewRandom().String()
+	tmp := uuid.NewString()
 	return &EtcDStorage{
 		url:       s.url,
 		db:        s.db,
@@ -145,7 +145,7 @@ func (s *EtcDStorage) Close() error {
 	return err
 }
 
-// Reopen tries to open (re-open) the database
+// Open returns the database
 func (s *EtcDStorage) Open() error {
 	if s.db != nil {
 		return nil

database/etcddb/transaction.go

@@ -67,8 +67,7 @@ func (t *transaction) Commit() (err error) {
 // Discard is safe to call after Commit(), it would be no-op
 func (t *transaction) Discard() {
 	t.ops = []clientv3.Op{}
-	t.tmpdb.Drop()
-	return
+	_ = t.tmpdb.Drop()
 }
 
 // transaction should implement database.Transaction

database/goleveldb/database.go

@@ -51,8 +51,8 @@ func RecoverDB(path string) error {
 		return err
 	}
 
-	db.Close()
-	stor.Close()
+	_ = db.Close()
+	_ = stor.Close()
 
 	return nil
 }

database/goleveldb/database_test.go

@@ -119,17 +119,17 @@ func (s *LevelDBSuite) TestDelete(c *C) {
 func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
@@ -161,7 +161,7 @@ func (s *LevelDBSuite) TestHasPrefix(c *C) {
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
 
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -180,8 +180,8 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	c.Assert(err, IsNil)
 
 	batch := s.db.CreateBatch()
-	batch.Put(key2, value2)
-	batch.Delete(key)
+	_ = batch.Put(key2, value2)
+	_ = batch.Delete(key)
 
 	v, err := s.db.Get(key)
 	c.Check(err, IsNil)
@@ -202,9 +202,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 }
 
 func (s *LevelDBSuite) TestCompactDB(c *C) {
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 
 	c.Check(s.db.CompactDB(), IsNil)
 }

deb/changes.go

@@ -60,14 +60,14 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	if err != nil {
 		return err
 	}
-	defer input.Close()
+	defer func() { _ = input.Close() }()
 
 	isClearSigned, err := verifier.IsClearSigned(input)
 	if err != nil {
 		return err
 	}
 
-	input.Seek(0, 0)
+	_, _ = input.Seek(0, 0)
 
 	if !isClearSigned && !acceptUnsigned {
 		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
@@ -79,7 +79,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		if err != nil {
 			return err
 		}
-		input.Seek(0, 0)
+		_, _ = input.Seek(0, 0)
 
 		c.SignatureKeys = keyInfo.GoodKeys
 	}
@@ -91,7 +91,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		if err != nil {
 			return err
 		}
-		defer text.Close()
+		defer func() { _ = text.Close() }()
 	} else {
 		text = input
 	}
@@ -307,7 +307,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -315,7 +315,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -334,7 +334,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -354,7 +354,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 				failedFiles = append(failedFiles, path)
 				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
 					changes.ChangesName, changes.SignatureKeys, err)
-				changes.Cleanup()
+				_ = changes.Cleanup()
 				continue
 			}
 		}

deb/changes_test.go

@@ -51,7 +51,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 
 func (s *ChangesSuite) TearDownTest(c *C) {
 	s.progress.Shutdown()
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -108,13 +108,13 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 
 	for _, path := range origFailedFiles {
 		filename := filepath.Join(s.Dir, filepath.Base(path))
-		utils.CopyFile(path, filename)
+		_ = utils.CopyFile(path, filename)
 		expectedFailedFiles = append(expectedFailedFiles, filename)
 	}
 
 	for _, path := range origProcessedFiles {
 		filename := filepath.Join(s.Dir, filepath.Base(path))
-		utils.CopyFile(path, filename)
+		_ = utils.CopyFile(path, filename)
 		expectedProcessedFiles = append(expectedProcessedFiles, filename)
 	}
 

deb/checksum_collection_test.go

@@ -28,7 +28,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
 }
 
 func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *ChecksumCollectionSuite) TestFlow(c *C) {

deb/contents.go

@@ -7,7 +7,7 @@ import (
 	"io"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 )
 
 // ContentsIndex calculates mapping from files to packages, with sorting and aggregation
@@ -20,7 +20,7 @@ type ContentsIndex struct {
 func NewContentsIndex(db database.Storage) *ContentsIndex {
 	return &ContentsIndex{
 		db:     db,
-		prefix: []byte(uuid.New()),
+		prefix: []byte(uuid.NewString()),
 	}
 }
 

deb/deb.go

@@ -17,7 +17,7 @@ import (
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/kjk/lzma"
 	"github.com/klauspost/compress/zstd"
-	"github.com/smira/go-xz"
+	xz "github.com/smira/go-xz"
 )
 
 // Source kinds
@@ -35,7 +35,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
+	defer func() { _ = file.Close() }()
 
 	library := ar.NewReader(file)
 	for {
@@ -66,14 +66,14 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
 				}
-				defer ungzip.Close()
+				defer func() { _ = ungzip.Close() }()
 				tarInput = ungzip
 			case "control.tar.xz":
 				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
 				}
-				defer unxz.Close()
+				defer func() { _ = unxz.Close() }()
 				tarInput = unxz
 			case "control.tar.zst":
 				unzstd, err := zstd.NewReader(bufReader)
@@ -116,10 +116,10 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
+	defer func() { _ = file.Close() }()
 
 	isClearSigned, err := verifier.IsClearSigned(file)
-	file.Seek(0, 0)
+	_, _ = file.Seek(0, 0)
 
 	if err != nil {
 		return nil, err
@@ -132,7 +132,7 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		if err != nil {
 			return nil, err
 		}
-		defer text.Close()
+		defer func() { _ = text.Close() }()
 	} else {
 		text = file
 	}
@@ -181,7 +181,7 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 					if err != nil {
 						return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
 					}
-					defer ungzip.Close()
+					defer func() { _ = ungzip.Close() }()
 					tarInput = ungzip
 				}
 			case "data.tar.bz2":
@@ -191,11 +191,11 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
 				}
-				defer unxz.Close()
+				defer func() { _ = unxz.Close() }()
 				tarInput = unxz
 			case "data.tar.lzma":
 				unlzma := lzma.NewReader(bufReader)
-				defer unlzma.Close()
+				defer func() { _ = unlzma.Close() }()
 				tarInput = unlzma
 			case "data.tar.zst":
 				unzstd, err := zstd.NewReader(bufReader)

deb/find_dangling.go

@@ -0,0 +1,45 @@
+package deb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+// FindDanglingReferences finds references that exist in the given PackageRefList, but not in the given PackageCollection.
+// It returns all such references, so they can be removed from the database.
+func FindDanglingReferences(reflist *PackageRefList, packages *PackageCollection) (dangling *PackageRefList, err error) {
+	dangling = &PackageRefList{}
+
+	err = reflist.ForEach(func(key []byte) error {
+		ok, err := isDangling(packages, key)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			dangling.Refs = append(dangling.Refs, key)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return dangling, nil
+}
+
+func isDangling(packages *PackageCollection, key []byte) (bool, error) {
+	_, err := packages.ByKey(key)
+	if errors.Is(err, database.ErrNotFound) {
+		return true, nil
+	}
+
+	if err != nil {
+		return false, fmt.Errorf("get reference %q: %w", key, err)
+	}
+
+	return false, nil
+}

deb/find_dangling_test.go

@@ -0,0 +1,46 @@
+package deb_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+	"github.com/aptly-dev/aptly/deb"
+)
+
+func TestFindDanglingReferences(t *testing.T) {
+	reflist := deb.NewPackageRefList()
+	reflist.Refs = [][]byte{[]byte("P existing 1.2.3"), []byte("P dangling 1.2.3")}
+
+	db, _ := goleveldb.NewOpenDB(t.TempDir())
+	packages := deb.NewPackageCollection(db)
+
+	if err := packages.Update(&deb.Package{Name: "existing", Version: "1.2.3"}); err != nil {
+		t.Fatal(err)
+	}
+
+	dangling, err := deb.FindDanglingReferences(reflist, packages)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exp := &deb.PackageRefList{
+		Refs: [][]byte{[]byte("P dangling 1.2.3")},
+	}
+
+	compareRefs(t, exp, dangling)
+}
+
+func compareRefs(t *testing.T, exp, got *deb.PackageRefList) {
+	t.Helper()
+
+	if len(exp.Refs) != len(got.Refs) {
+		t.Fatalf("refs length mismatch: exp %d, got %d", len(exp.Refs), len(got.Refs))
+	}
+
+	for i := range exp.Refs {
+		if !bytes.Equal(exp.Refs[i], got.Refs[i]) {
+			t.Fatalf("refs do not match: exp %q, got %q", exp.Refs[i], got.Refs[i])
+		}
+	}
+}

deb/format.go

@@ -288,8 +288,11 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			lastField = canonicalCase(parts[0])
 			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
-				stanza[lastField] = parts[1]
-				if parts[1] != "" {
+				// Trim trailing whitespace from the inline value so that
+				// "Package-List: " does not add empty line
+				inlineVal := strings.TrimRight(parts[1], " \t")
+				stanza[lastField] = inlineVal
+				if inlineVal != "" {
 					stanza[lastField] += "\n"
 				}
 			} else {

deb/format_test.go

@@ -128,6 +128,35 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+// Sources may contain "Package-List: " with a trailing space.
+// That trailing space must not be preserved and re-emitted
+// as a spurious blank continuation line when the stanza is written back out.
+func (s *ControlFileSuite) TestPackageListTrailingSpace(c *C) {
+	input := "Package-List: \n" +
+		" bash deb shells required arch=any\n" +
+		" bash-doc deb doc optional arch=all\n"
+
+	r := NewControlFileReader(bytes.NewBufferString(input), false, false)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(stanza["Package-List"], Equals,
+		" bash deb shells required arch=any\n"+
+			" bash-doc deb doc optional arch=all\n")
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, true, false, false)
+	c.Assert(err, IsNil)
+	c.Assert(w.Flush(), IsNil)
+
+	written := buf.String()
+	c.Assert(strings.Contains(written, "Package-List:\n \n"), Equals, false,
+		Commentf("spurious blank continuation line found in written output:\n%s", written))
+	c.Assert(strings.Contains(written, "Package-List:\n bash"), Equals, true,
+		Commentf("expected Package-List entries not found in written output:\n%s", written))
+}
+
 func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
 	s.reader = bytes.NewBufferString(installerFile)
 	r := NewControlFileReader(s.reader, false, true)
@@ -163,7 +192,7 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 func (s *ControlFileSuite) TestLongFields(c *C) {
 	f, err := os.Open("long.stanza")
 	c.Assert(err, IsNil)
-	defer f.Close()
+	defer func() { _ = f.Close() }()
 
 	r := NewControlFileReader(f, false, false)
 	stanza, e := r.ReadStanza()

deb/graph.go

@@ -12,15 +12,15 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	var err error
 
 	graph := gographviz.NewEscape()
-	graph.SetDir(true)
-	graph.SetName("aptly")
+	_ = graph.SetDir(true)
+	_ = graph.SetName("aptly")
 
 	var labelStart string
 	var labelEnd string
 
 	switch layout {
 	case "vertical":
-		graph.AddAttr("aptly", "rankdir", "LR")
+		_ = graph.AddAttr("aptly", "rankdir", "LR")
 		labelStart = ""
 		labelEnd = ""
 	case "horizontal":
@@ -38,7 +38,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			return e
 		}
 
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
@@ -60,7 +60,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			return e
 		}
 
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
@@ -75,7 +75,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		return nil, err
 	}
 
-	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+	_ = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
 		existingNodes[snapshot.UUID] = true
 		return nil
 	})
@@ -91,7 +91,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			description = "Snapshot from repo"
 		}
 
-		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+		_ = graph.AddNode("aptly", snapshot.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
@@ -103,7 +103,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {
-					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+					_ = graph.AddEdge(uuid, snapshot.UUID, true, nil)
 				}
 			}
 		}
@@ -114,8 +114,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		return nil, err
 	}
 
-	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+	_ = collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
@@ -127,7 +127,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		for _, uuid := range repo.Sources {
 			_, exists := existingNodes[uuid]
 			if exists {
-				graph.AddEdge(uuid, repo.UUID, true, nil)
+				_ = graph.AddEdge(uuid, repo.UUID, true, nil)
 			}
 		}
 

deb/index_files.go

@@ -59,24 +59,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		if file.discardable {
 			return nil
 		}
-		file.BufWriter()
+		_, _ = file.BufWriter()
 	}
 
 	err := file.w.Flush()
 	if err != nil {
-		file.tempFile.Close()
+		_ = file.tempFile.Close()
 		return fmt.Errorf("unable to write to index file: %s", err)
 	}
 
 	if file.compressable {
 		err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
 		if err != nil {
-			file.tempFile.Close()
+			_ = file.tempFile.Close()
 			return fmt.Errorf("unable to compress index file: %s", err)
 		}
 	}
 
-	file.tempFile.Close()
+	_ = file.tempFile.Close()
 
 	exts := []string{""}
 	cksumExts := exts
@@ -220,11 +220,11 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
 				// If we managed to resolve the link target: delete it. This is the
 				// oldest physical index file we no longer need. Once we drop our
 				// old symlink we'll essentially forget about it existing at all.
-				file.parent.publishedStorage.Remove(linkTarget)
+				_ = file.parent.publishedStorage.Remove(linkTarget)
 			}
-			file.parent.publishedStorage.Remove(oldIndexPath)
+			_ = file.parent.publishedStorage.Remove(oldIndexPath)
 		}
-		file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
+		_ = file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
 	}
 
 	// create symlink

deb/list.go

@@ -438,7 +438,7 @@ func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
 	result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
 	for _, pkg := range l.packages {
 		if q.Matches(pkg) {
-			result.Add(pkg)
+			_ = result.Add(pkg)
 		}
 	}
 
@@ -456,7 +456,7 @@ func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageLi
 
 	pkg := l.packages["P"+arch+" "+name+" "+version]
 	if pkg != nil {
-		result.Add(pkg)
+		_ = result.Add(pkg)
 	}
 
 	return
@@ -598,6 +598,7 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) {
 					//
 					// when follow-all-variants is enabled, we need to try to expand anyway,
 					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					// FIXME: do not search twice
 					if result.Search(dep, false, true) != nil {
 						if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
 							options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true))

deb/list_test.go

@@ -96,7 +96,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 		{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
 	}
 	for _, p := range s.packages {
-		s.il.Add(p)
+		_ = s.il.Add(p)
 	}
 	s.il.PrepareIndex()
 
@@ -110,7 +110,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
 		{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
 	}
 	for _, p := range s.packages2 {
-		s.il2.Add(p)
+		_ = s.il2.Add(p)
 	}
 	s.il2.PrepareIndex()
 
@@ -202,8 +202,8 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
 }
 
 func (s *PackageListSuite) TestForeach(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
 
 	Len := 0
 	err := s.list.ForEach(func(*Package) error {
@@ -232,21 +232,21 @@ func (s *PackageListSuite) TestIndex(c *C) {
 }
 
 func (s *PackageListSuite) TestAppend(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
 
 	err := s.list.Append(s.il)
 	c.Check(err, IsNil)
 	c.Check(s.list.Len(), Equals, 16)
 
 	list := NewPackageList()
-	list.Add(s.p4)
+	_ = list.Add(s.p4)
 
 	err = s.list.Append(list)
 	c.Check(err, ErrorMatches, "package already exists and is different: .*")
 
 	s.list.PrepareIndex()
-	c.Check(func() { s.list.Append(s.il) }, Panics, "Append not supported when indexed")
+	c.Check(func() { _ = s.list.Append(s.il) }, Panics, "Append not supported when indexed")
 }
 
 func (s *PackageListSuite) TestSearch(c *C) {
@@ -312,7 +312,7 @@ func (s *PackageListSuite) TestSearch(c *C) {
 
 func (s *PackageListSuite) TestFilter(c *C) {
 	c.Check(func() {
-		s.list.Filter(FilterOptions{
+		_, _ = s.list.Filter(FilterOptions{
 			Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}},
 		})
 	}, Panics, "list not indexed, can't filter")
@@ -479,7 +479,7 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
 		{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
 
 	for _, p := range s.sourcePackages {
-		s.il.Add(p)
+		_ = s.il.Add(p)
 	}
 
 	missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)

deb/local.go

@@ -7,7 +7,7 @@ import (
 	"log"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 	"github.com/ugorji/go/codec"
 )
 
@@ -32,7 +32,7 @@ type LocalRepo struct {
 // NewLocalRepo creates new instance of Debian local repository
 func NewLocalRepo(name string, comment string) *LocalRepo {
 	return &LocalRepo{
-		UUID:    uuid.New(),
+		UUID:    uuid.NewString(),
 		Name:    name,
 		Comment: comment,
 	}
@@ -69,7 +69,7 @@ func (repo *LocalRepo) Encode() []byte {
 	var buf bytes.Buffer
 
 	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(repo)
+	_ = encoder.Encode(repo)
 
 	return buf.Bytes()
 }
@@ -116,7 +116,7 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
 		return result
 	}
 
-	collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
+	_ = collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
 		r := &LocalRepo{}
 		if err := r.Decode(blob); err != nil {
 			log.Printf("Error decoding local repo: %s\n", err)
@@ -159,15 +159,17 @@ func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
 // Update stores updated information about repo in DB
 func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
 	batch := collection.db.CreateBatch()
-	batch.Put(repo.Key(), repo.Encode())
+	_ = batch.Put(repo.Key(), repo.Encode())
 	if repo.packageRefs != nil {
-		batch.Put(repo.RefKey(), repo.packageRefs.Encode())
+		_ = batch.Put(repo.RefKey(), repo.packageRefs.Encode())
 	}
 	return batch.Write()
 }
 
 // LoadComplete loads additional information for local repo
 func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
+	repo.packageRefs = &PackageRefList{}
+
 	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
@@ -176,7 +178,6 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 		return err
 	}
 
-	repo.packageRefs = &PackageRefList{}
 	return repo.packageRefs.Decode(encoded)
 }
 
@@ -247,7 +248,7 @@ func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
 	delete(collection.cache, repo.UUID)
 
 	batch := collection.db.CreateBatch()
-	batch.Delete(repo.Key())
-	batch.Delete(repo.RefKey())
+	_ = batch.Delete(repo.Key())
+	_ = batch.Delete(repo.RefKey())
 	return batch.Write()
 }

deb/local_test.go

@@ -21,8 +21,8 @@ var _ = Suite(&LocalRepoSuite{})
 func (s *LocalRepoSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.list = NewPackageList()
-	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
-	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
+	_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
+	_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 
@@ -31,7 +31,7 @@ func (s *LocalRepoSuite) SetUpTest(c *C) {
 }
 
 func (s *LocalRepoSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *LocalRepoSuite) TestString(c *C) {
@@ -88,14 +88,14 @@ func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
 	s.collection = NewLocalRepoCollection(s.db)
 
 	s.list = NewPackageList()
-	s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
-	s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
+	_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
+	_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
 
 	s.reflist = NewPackageRefListFromPackageList(s.list)
 }
 
 func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
@@ -133,6 +133,18 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(r.String(), Equals, repo.String())
 }
 
+func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) {
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	r, err := s.collection.ByName("local1")
+	c.Assert(err, IsNil)
+
+	c.Assert(s.collection.LoadComplete(r), IsNil)
+	c.Assert(r.packageRefs, NotNil)
+	c.Assert(r.NumPackages(), Equals, 0)
+}
+
 func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Update(repo), IsNil)
@@ -156,7 +168,7 @@ func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 
 func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
-	s.collection.Add(repo)
+	_ = s.collection.Add(repo)
 
 	count := 0
 	err := s.collection.ForEach(func(*LocalRepo) error {
@@ -178,10 +190,10 @@ func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
 
 func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
 	repo1 := NewLocalRepo("local1", "Comment 1")
-	s.collection.Add(repo1)
+	_ = s.collection.Add(repo1)
 
 	repo2 := NewLocalRepo("local2", "Comment 2")
-	s.collection.Add(repo2)
+	_ = s.collection.Add(repo2)
 
 	r1, _ := s.collection.ByUUID(repo1.UUID)
 	c.Check(r1, Equals, repo1)
@@ -208,6 +220,6 @@ func (s *LocalRepoCollectionSuite) TestDropNonExisting(c *C) {
 	_, err := s.collection.ByUUID(repo.UUID)
 	c.Check(err, ErrorMatches, "local repo .* not found")
 
-	err = s.collection.Drop(repo)
+	_ = s.collection.Drop(repo)
 	c.Check(s.collection.Drop(repo), ErrorMatches, "local repo not found")
 }

deb/package.go

@@ -565,7 +565,7 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl
 		}
 		return nil, err
 	}
-	defer reader.Close()
+	defer func() { _ = reader.Close() }()
 
 	contents, err := GetContentsFromDeb(reader, file.Filename)
 	if err != nil {

deb/package_collection.go

@@ -309,7 +309,7 @@ func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList)
 		}
 
 		if q.Matches(pkg) {
-			result.Add(pkg)
+			_ = result.Add(pkg)
 		}
 	}
 
@@ -337,7 +337,7 @@ func (collection *PackageCollection) SearchByKey(arch, name, version string) (re
 		}
 
 		if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
-			result.Add(pkg)
+			_ = result.Add(pkg)
 		}
 	}
 

deb/package_collection_test.go

@@ -23,7 +23,7 @@ func (s *PackageCollectionSuite) SetUpTest(c *C) {
 }
 
 func (s *PackageCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *PackageCollectionSuite) TestUpdate(c *C) {
@@ -67,7 +67,7 @@ func (s *PackageCollectionSuite) TestByKey(c *C) {
 
 func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) {
 	key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1")
-	s.db.Put(key, old0_3Package)
+	_ = s.db.Put(key, old0_3Package)
 
 	p, err := s.collection.ByKey(key)
 	c.Check(err, IsNil)

deb/package_files.go

@@ -64,7 +64,7 @@ func (files PackageFiles) Hash() uint64 {
 
 	for _, f := range files {
 		h.Write([]byte(f.Filename))
-		binary.Write(h, binary.BigEndian, f.Checksums.Size)
+		_ = binary.Write(h, binary.BigEndian, f.Checksums.Size)
 		h.Write([]byte(f.Checksums.MD5))
 		h.Write([]byte(f.Checksums.SHA1))
 		h.Write([]byte(f.Checksums.SHA256))

deb/package_files_test.go

@@ -1,7 +1,7 @@
 package deb
 
 import (
-	"io/ioutil"
+	"os"
 	"path/filepath"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -39,7 +39,7 @@ func (s *PackageFilesSuite) TestVerify(c *C) {
 	c.Check(result, Equals, false)
 
 	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
 	s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
 

deb/package_test.go

@@ -2,7 +2,7 @@ package deb
 
 import (
 	"bytes"
-	"io/ioutil"
+	"os"
 	"path/filepath"
 	"regexp"
 
@@ -395,7 +395,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
 	p := NewPackageFromControlFile(s.stanza)
 
 	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+	c.Assert(os.WriteFile(tmpFilepath, nil, 0777), IsNil)
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
@@ -434,7 +434,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
 	})
 
 	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 
 	list, err = p.DownloadList(packagePool, cs)
@@ -449,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
 	cs := files.NewMockChecksumStorage()
 
 	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
+	c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
 
 	p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
 

deb/ppa.go

@@ -28,6 +28,11 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 		}
 	}
 
+	baseurl := config.PpaBaseURL
+	if baseurl == "" {
+		baseurl = "http://ppa.launchpad.net"
+	}
+
 	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
@@ -39,7 +44,7 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 
 	distribution = codename
 	components = []string{"main"}
-	url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID)
+	url = fmt.Sprintf("%s/%s/%s/%s", baseurl, matches[1], matches[2], distributorID)
 
 	return
 }

deb/publish.go

@@ -9,10 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 	"github.com/ugorji/go/codec"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -354,7 +355,7 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
 func NewPublishedRepo(storage, prefix, distribution string, architectures []string,
 	components []string, sources []interface{}, collectionFactory *CollectionFactory, multiDist bool) (*PublishedRepo, error) {
 	result := &PublishedRepo{
-		UUID:          uuid.New(),
+		UUID:          uuid.NewString(),
 		Storage:       storage,
 		Architectures: architectures,
 		Sources:       make(map[string]string),
@@ -603,6 +604,15 @@ func (p *PublishedRepo) Key() []byte {
 	return []byte("U" + p.StoragePrefix() + ">>" + p.Distribution)
 }
 
+// PrefixPoolLockKey returns the task-queue resource key that serialises all
+// publish operations sharing the same pool directory under storagePrefix.
+// It must be held whenever a non-MultiDist publish may read or clean the
+// shared pool, to prevent concurrent cleanup runs from deleting each other's
+// files.  See docs/Resource-Locking.md for the full key-namespace table.
+func PrefixPoolLockKey(storagePrefix string) string {
+	return "P" + storagePrefix
+}
+
 // RefKey is a unique id for package reference list
 func (p *PublishedRepo) RefKey(component string) []byte {
 	return []byte("E" + p.UUID + component)
@@ -631,7 +641,7 @@ func (p *PublishedRepo) Components() []string {
 	return result
 }
 
-// Components returns sorted list of published repo source names
+// SourceNames returns sorted list of published repo source names
 func (p *PublishedRepo) SourceNames() []string {
 	var sources = []string{}
 
@@ -702,7 +712,7 @@ func (p *PublishedRepo) Encode() []byte {
 	var buf bytes.Buffer
 
 	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(p)
+	_ = encoder.Encode(p)
 
 	return buf.Bytes()
 }
@@ -814,9 +824,12 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
-	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
+	err = publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
 	if err != nil {
 		return err
 	}
@@ -884,7 +897,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	if err != nil {
 		return err
 	}
-	defer os.RemoveAll(tempDir)
+	defer func() { _ = os.RemoveAll(tempDir) }()
 
 	indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash, p.SkipBz2)
 
@@ -970,7 +983,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 								contentIndexesMap[key] = contentIndex
 							}
 
-							contentIndex.Push(qualifiedName, contents, batch)
+							_ = contentIndex.Push(qualifiedName, contents, batch)
 						}
 					}
 
@@ -1126,7 +1139,15 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.GetSuite()
 	release["Codename"] = p.GetCodename()
-	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
+	datetimeFormat := "Mon, 2 Jan 2006 15:04:05 MST"
+
+	publishDate := time.Now().UTC()
+	if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
+		if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil {
+			publishDate = time.Unix(sec, 0).UTC()
+		}
+	}
+	release["Date"] = publishDate.Format(datetimeFormat)
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	if p.AcquireByHash {
 		release["Acquire-By-Hash"] = "yes"
@@ -1182,7 +1203,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 // It can remove prefix fully, and part of pool (for specific component)
 func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool,
 	removePoolComponents []string, progress aptly.Progress) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
 	// I. Easy: remove whole prefix (meta+packages)
 	if removePrefix {
@@ -1195,7 +1219,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 	}
 
 	// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
-	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
+	err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
 	if err != nil {
 		return err
 	}
@@ -1275,11 +1299,11 @@ func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *
 // Update stores updated information about repo in DB
 func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
 	batch := collection.db.CreateBatch()
-	batch.Put(repo.Key(), repo.Encode())
+	_ = batch.Put(repo.Key(), repo.Encode())
 
 	if repo.SourceKind == SourceLocalRepo {
 		for component, item := range repo.sourceItems {
-			batch.Put(repo.RefKey(component), item.packageRefs.Encode())
+			_ = batch.Put(repo.RefKey(component), item.packageRefs.Encode())
 		}
 	}
 	return batch.Write()
@@ -1324,7 +1348,7 @@ func (collection *PublishedRepoCollection) LoadShallow(repo *PublishedRepo, coll
 
 // LoadComplete loads complete information on the sources of the repo *and* their packages
 func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
-	collection.LoadShallow(repo, collectionFactory)
+	_ = collection.LoadShallow(repo, collectionFactory)
 
 	if repo.SourceKind == SourceSnapshot {
 		for _, item := range repo.sourceItems {
@@ -1502,7 +1526,7 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 						return nil, err
 					}
 
-					packageList.ForEach(func(p *Package) error {
+					_ = packageList.ForEach(func(p *Package) error {
 						poolDir, err := p.PoolDirectory()
 						if err != nil {
 							return err
@@ -1522,6 +1546,55 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 	return referencedFiles, nil
 }
 
+// CleanupAfterMultiDistToggle cleans up stale pool files left behind when the
+// MultiDist flag is toggled on a published repository.
+//
+//   - false→true: Publish() wrote packages into pool/<distribution>/<component>/
+//     but the old flat pool/<component>/ files were not removed because
+//     CleanupPrefixComponentFiles only scans the new MultiDist tree.
+//     A second pass with MultiDist=false cleans the legacy flat layout by
+//     reusing the existing orphan-detection logic (the repo is now MultiDist=true
+//     so it is excluded from the referenced-files scan, making its old pool
+//     entries appear orphaned).
+//
+//   - true→false: Publish() wrote packages into pool/<component>/ but the old
+//     per-distribution pool/<distribution>/<component>/ directories were not
+//     removed.  The orphan-detection approach cannot be used here because the
+//     repo's RefList still contains all packages (they just moved locations).
+//     Instead we directly remove each pool/<distribution>/<component>/ directory.
+//     This is safe because per-distribution pool dirs are exclusive to a single
+//     prefix+distribution combination — no other published repo can share them.
+func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(publishedStorageProvider aptly.PublishedStorageProvider,
+	published *PublishedRepo, prevMultiDist bool, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	if prevMultiDist == published.MultiDist {
+		return nil
+	}
+
+	if !prevMultiDist && published.MultiDist {
+		// false→true: use orphan-detection via the existing cleanup, but with
+		// MultiDist temporarily set to false so it scans the flat pool layout.
+		legacy := *published
+		legacy.MultiDist = false
+		return collection.CleanupPrefixComponentFiles(publishedStorageProvider, &legacy, cleanComponents, collectionFactory, progress)
+	}
+
+	// true→false: directly remove the per-distribution pool directories.
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
+	for _, component := range cleanComponents {
+		poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component)
+		if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil {
+			return err
+		}
+	}
+	// Remove the distribution-level pool dir if it is now empty.
+	distPoolDir := filepath.Join(published.Prefix, "pool", published.Distribution)
+	_ = publishedStorage.RemoveDirs(distPoolDir, progress)
+	return nil
+}
+
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
 	published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
@@ -1535,7 +1608,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
 	distribution := published.Distribution
 
 	rootPath := filepath.Join(prefix, "dists", distribution)
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
 
 	sort.Strings(cleanComponents)
 	publishedComponents := published.Components()
@@ -1575,7 +1651,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
 				return err
 			}
 
-			packageList.ForEach(func(p *Package) error {
+			_ = packageList.ForEach(func(p *Package) error {
 				poolDir, err := p.PoolDirectory()
 				if err != nil {
 					return err
@@ -1709,10 +1785,10 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
 	}
 
 	batch := collection.db.CreateBatch()
-	batch.Delete(repo.Key())
+	_ = batch.Delete(repo.Key())
 
 	for _, component := range repo.Components() {
-		batch.Delete(repo.RefKey(component))
+		_ = batch.Delete(repo.RefKey(component))
 	}
 
 	return batch.Write()

deb/publish_bench_test.go

@@ -19,13 +19,13 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	defer os.RemoveAll(tmpDir)
+	defer func() { _ = os.RemoveAll(tmpDir) }()
 
 	db, err := goleveldb.NewOpenDB(tmpDir)
 	if err != nil {
 		b.Fatal(err)
 	}
-	defer db.Close()
+	defer func() { _ = db.Close() }()
 
 	factory := NewCollectionFactory(db)
 	packageCollection := factory.PackageCollection()
@@ -49,7 +49,7 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 				Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex),
 			}})
 
-			packageCollection.UpdateInTransaction(p, transaction)
+			_ = packageCollection.UpdateInTransaction(p, transaction)
 			sharedRefs.Refs = append(sharedRefs.Refs, p.Key(""))
 		}
 
@@ -78,7 +78,7 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 				Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex),
 			}})
 
-			packageCollection.UpdateInTransaction(p, transaction)
+			_ = packageCollection.UpdateInTransaction(p, transaction)
 			refs.Refs = append(refs.Refs, p.Key(""))
 		}
 
@@ -92,16 +92,16 @@ func BenchmarkListReferencedFiles(b *testing.B) {
 		repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex)
 		repo.DefaultComponent = defaultComponent
 		repo.UpdateRefList(refs.Merge(sharedRefs, false, true))
-		repoCollection.Add(repo)
+		_ = repoCollection.Add(repo)
 
 		publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false)
 		if err != nil {
 			b.Fatal(err)
 		}
-		publishCollection.Add(publish)
+		_ = publishCollection.Add(publish)
 	}
 
-	db.CompactDB()
+	_ = db.CompactDB()
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {

deb/publish_test.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sort"
@@ -51,23 +50,23 @@ func (n *NullSigner) SetPassphrase(passphrase, passphraseFile string) {
 }
 
 func (n *NullSigner) DetachedSign(source string, destination string) error {
-	return ioutil.WriteFile(destination, []byte{}, 0644)
+	return os.WriteFile(destination, []byte{}, 0644)
 }
 
 func (n *NullSigner) ClearSign(source string, destination string) error {
-	return ioutil.WriteFile(destination, []byte{}, 0644)
+	return os.WriteFile(destination, []byte{}, 0644)
 }
 
 type FakeStorageProvider struct {
 	storages map[string]aptly.PublishedStorage
 }
 
-func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+func (p *FakeStorageProvider) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	storage, ok := p.storages[name]
 	if !ok {
-		panic(fmt.Sprintf("unknown storage: %#v", name))
+		return nil, fmt.Errorf("unknown storage: %#v", name)
 	}
-	return storage
+	return storage, nil
 }
 
 type PublishedRepoSuite struct {
@@ -104,7 +103,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 	s.cs = files.NewMockChecksumStorage()
 
 	tmpFilepath := filepath.Join(c.MkDir(), "file")
-	c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
+	c.Assert(os.WriteFile(tmpFilepath, nil, 0777), IsNil)
 
 	var err error
 	s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs)
@@ -118,22 +117,22 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 
 	repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
 	repo.packageRefs = s.reflist
-	s.factory.RemoteRepoCollection().Add(repo)
+	_ = s.factory.RemoteRepoCollection().Add(repo)
 
 	s.localRepo = NewLocalRepo("local1", "comment1")
 	s.localRepo.packageRefs = s.reflist
-	s.factory.LocalRepoCollection().Add(s.localRepo)
+	_ = s.factory.LocalRepoCollection().Add(s.localRepo)
 
 	s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
-	s.factory.SnapshotCollection().Add(s.snapshot)
+	_ = s.factory.SnapshotCollection().Add(s.snapshot)
 
 	s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
-	s.factory.SnapshotCollection().Add(s.snapshot2)
+	_ = s.factory.SnapshotCollection().Add(s.snapshot2)
 
 	s.packageCollection = s.factory.PackageCollection()
-	s.packageCollection.Update(s.p1)
-	s.packageCollection.Update(s.p2)
-	s.packageCollection.Update(s.p3)
+	_ = s.packageCollection.Update(s.p1)
+	_ = s.packageCollection.Update(s.p2)
+	_ = s.packageCollection.Update(s.p3)
 
 	s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
 	s.repo.SkipContents = true
@@ -152,7 +151,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedRepoSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
@@ -179,12 +178,12 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
 	c.Check(s.repo3.RefList("main").Len(), Equals, 3)
 	c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
 
-	c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources")
+	c.Check(func() { _, _ = NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources")
 	c.Check(func() {
-		NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
+		_, _ = NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
 	}, PanicMatches, "sources and components should be equal in size")
 	c.Check(func() {
-		NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false)
+		_, _ = NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false)
 	}, PanicMatches, "interface conversion:.*")
 
 	_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
@@ -337,7 +336,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
 
 	s.localRepo.DefaultDistribution = "precise"
 	s.localRepo.DefaultComponent = "contrib"
-	s.factory.LocalRepoCollection().Update(s.localRepo)
+	_ = s.factory.LocalRepoCollection().Update(s.localRepo)
 
 	repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
 	c.Check(err, IsNil)
@@ -434,6 +433,47 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
+func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) {
+	// Test with SOURCE_DATE_EPOCH set
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC
+	c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
+func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) {
+	// Test with invalid SOURCE_DATE_EPOCH (should fallback to current time)
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "invalid")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH)
+	c.Check(st["Date"], Not(Equals), "")
+	c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
 	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)
@@ -526,8 +566,8 @@ func (s *PublishedRepoSuite) TestPublishedRepoRevision(c *C) {
 	bytes, err := json.Marshal(revision)
 	c.Assert(err, IsNil)
 
-	json_expected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}`
-	c.Assert(string(bytes), Equals, json_expected)
+	jsonExpected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}`
+	c.Assert(string(bytes), Equals, jsonExpected)
 
 	c.Assert(s.repo2.DropRevision(), DeepEquals, revision)
 	c.Assert(s.repo2.Revision, IsNil)
@@ -564,11 +604,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 	sort.Sort(snap2Refs)
 	s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2")
 
-	s.snapshotCollection.Add(s.snap1)
-	s.snapshotCollection.Add(s.snap2)
+	_ = s.snapshotCollection.Add(s.snap1)
+	_ = s.snapshotCollection.Add(s.snap2)
 
 	s.localRepo = NewLocalRepo("local1", "comment1")
-	s.factory.LocalRepoCollection().Add(s.localRepo)
+	_ = s.factory.LocalRepoCollection().Add(s.localRepo)
 
 	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
 	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false)
@@ -580,7 +620,7 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
@@ -677,7 +717,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 	var buf bytes.Buffer
 
 	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(&old)
+	_ = encoder.Encode(&old)
 
 	c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
 	c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
@@ -695,7 +735,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
 }
 
 func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
-	s.collection.Add(s.repo1)
+	_ = s.collection.Add(s.repo1)
 
 	count := 0
 	err := s.collection.ForEach(func(*PublishedRepo) error {
@@ -755,9 +795,12 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 	})
 
 	snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
-	s.snapshotCollection.Add(snap3)
+	_ = s.snapshotCollection.Add(snap3)
 
-	// Ensure that adding a second publish point with matching files doesn't give duplicate results.
+	// When a second publish point references the same package (snap3 is a clone of snap2,
+	// both containing p3/lonely-strangers), listReferencedFilesByComponent deduplicates by
+	// package ref so the file appears only once.  StrSlicesSubstract handles a single entry
+	// correctly, so no duplicate is needed for cleanup safety.
 	repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(s.collection.Add(repo3), IsNil)
@@ -772,7 +815,9 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 			"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			"a/alien-arena/mars-invaders_7.40-2_i386.deb",
 		},
-		"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
+		"main": {
+			"a/alien-arena/lonely-strangers_7.40-2_i386.deb",
+		},
 	})
 }
 
@@ -799,7 +844,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 
 	s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
 
-	s.snapshotCollection.Add(s.snap1)
+	_ = s.snapshotCollection.Add(s.snap1)
 
 	s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
 	s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
@@ -808,26 +853,26 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 	s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
 
 	s.collection = s.factory.PublishedRepoCollection()
-	s.collection.Add(s.repo1)
-	s.collection.Add(s.repo2)
-	s.collection.Add(s.repo3)
-	s.collection.Add(s.repo4)
-	s.collection.Add(s.repo5)
+	_ = s.collection.Add(s.repo1)
+	_ = s.collection.Add(s.repo2)
+	_ = s.collection.Add(s.repo3)
+	_ = s.collection.Add(s.repo4)
+	_ = s.collection.Add(s.repo5)
 
 	s.root = c.MkDir()
 	s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
-	s.publishedStorage.MkDir("ppa/dists/anaconda")
-	s.publishedStorage.MkDir("ppa/dists/meduza")
-	s.publishedStorage.MkDir("ppa/dists/osminog")
-	s.publishedStorage.MkDir("ppa/pool/main")
-	s.publishedStorage.MkDir("ppa/pool/contrib")
-	s.publishedStorage.MkDir("dists/anaconda")
-	s.publishedStorage.MkDir("pool/main")
+	_ = s.publishedStorage.MkDir("ppa/dists/anaconda")
+	_ = s.publishedStorage.MkDir("ppa/dists/meduza")
+	_ = s.publishedStorage.MkDir("ppa/dists/osminog")
+	_ = s.publishedStorage.MkDir("ppa/pool/main")
+	_ = s.publishedStorage.MkDir("ppa/pool/contrib")
+	_ = s.publishedStorage.MkDir("dists/anaconda")
+	_ = s.publishedStorage.MkDir("pool/main")
 
 	s.root2 = c.MkDir()
 	s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
-	s.publishedStorage2.MkDir("ppa/dists/osminog")
-	s.publishedStorage2.MkDir("ppa/pool/contrib")
+	_ = s.publishedStorage2.MkDir("ppa/dists/osminog")
+	_ = s.publishedStorage2.MkDir("ppa/pool/contrib")
 
 	s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
 		"":            s.publishedStorage,
@@ -835,11 +880,11 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
-	s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
+	_ = s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -853,7 +898,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
-	s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
+	_ = s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -867,7 +912,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
-	s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
+	_ = s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -881,7 +926,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
-	s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
+	_ = s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
@@ -895,7 +940,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
 }
 
 func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
-	s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
+	_ = s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
 
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)

deb/query.go

@@ -89,7 +89,7 @@ func (q *OrQuery) Fast(list PackageCatalog) bool {
 func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) {
 	if q.Fast(list) {
 		result = q.L.Query(list)
-		result.Append(q.R.Query(list))
+		_ = result.Append(q.R.Query(list))
 	} else {
 		result = list.Scan(q)
 	}
@@ -245,7 +245,7 @@ func (q *DependencyQuery) Query(list PackageCatalog) (result *PackageList) {
 	if q.Fast(list) {
 		result = NewPackageList()
 		for _, pkg := range list.Search(q.Dep, true, true) {
-			result.Add(pkg)
+			_ = result.Add(pkg)
 		}
 	} else {
 		result = list.Scan(q)

deb/reflist.go

@@ -54,7 +54,7 @@ func (l *PackageRefList) Swap(i, j int) {
 	l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i]
 }
 
-// Compare compares two refs in lexographical order
+// Less compares two refs in lexographical order
 func (l *PackageRefList) Less(i, j int) bool {
 	return bytes.Compare(l.Refs[i], l.Refs[j]) < 0
 }
@@ -64,7 +64,7 @@ func (l *PackageRefList) Encode() []byte {
 	var buf bytes.Buffer
 
 	encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
-	encoder.Encode(l)
+	_ = encoder.Encode(l)
 
 	return buf.Bytes()
 }
@@ -79,6 +79,9 @@ func (l *PackageRefList) Decode(input []byte) error {
 
 // ForEach calls handler for each package ref in list
 func (l *PackageRefList) ForEach(handler func([]byte) error) error {
+	if l == nil {
+		return nil
+	}
 	var err error
 	for _, p := range l.Refs {
 		err = handler(p)

deb/reflist_bench_test.go

@@ -42,6 +42,6 @@ func BenchmarkReflistDecode(b *testing.B) {
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		(&PackageRefList{}).Decode(data)
+		_ = (&PackageRefList{}).Decode(data)
 	}
 }

deb/reflist_test.go

@@ -46,26 +46,26 @@ func (s *PackageRefListSuite) SetUpTest(c *C) {
 func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
 	db, _ := goleveldb.NewOpenDB(c.MkDir())
 	coll := NewPackageCollection(db)
-	coll.Update(s.p1)
-	coll.Update(s.p3)
+	_ = coll.Update(s.p1)
+	_ = coll.Update(s.p3)
 
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
+	_ = s.list.Add(s.p5)
+	_ = s.list.Add(s.p6)
 
 	reflist := NewPackageRefListFromPackageList(s.list)
 
 	_, err := NewPackageListFromRefList(reflist, coll, nil)
 	c.Assert(err, ErrorMatches, "unable to load package with key.*")
 
-	coll.Update(s.p5)
-	coll.Update(s.p6)
+	_ = coll.Update(s.p5)
+	_ = coll.Update(s.p6)
 
 	list, err := NewPackageListFromRefList(reflist, coll, nil)
 	c.Assert(err, IsNil)
 	c.Check(list.Len(), Equals, 4)
-        c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
+	c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*")
 
 	list, err = NewPackageListFromRefList(nil, coll, nil)
 	c.Assert(err, IsNil)
@@ -73,10 +73,10 @@ func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) {
 }
 
 func (s *PackageRefListSuite) TestNewPackageRefList(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
+	_ = s.list.Add(s.p5)
+	_ = s.list.Add(s.p6)
 
 	reflist := NewPackageRefListFromPackageList(s.list)
 	c.Assert(reflist.Len(), Equals, 4)
@@ -90,10 +90,10 @@ func (s *PackageRefListSuite) TestNewPackageRefList(c *C) {
 }
 
 func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
+	_ = s.list.Add(s.p5)
+	_ = s.list.Add(s.p6)
 
 	reflist := NewPackageRefListFromPackageList(s.list)
 
@@ -105,10 +105,10 @@ func (s *PackageRefListSuite) TestPackageRefListEncodeDecode(c *C) {
 }
 
 func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
-	s.list.Add(s.p6)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
+	_ = s.list.Add(s.p5)
+	_ = s.list.Add(s.p6)
 
 	reflist := NewPackageRefListFromPackageList(s.list)
 
@@ -130,10 +130,21 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestForEachNilList(c *C) {
+	var l *PackageRefList
+	called := false
+	err := l.ForEach(func([]byte) error {
+		called = true
+		return nil
+	})
+	c.Assert(err, IsNil)
+	c.Assert(called, Equals, false)
+}
+
 func (s *PackageRefListSuite) TestHas(c *C) {
-	s.list.Add(s.p1)
-	s.list.Add(s.p3)
-	s.list.Add(s.p5)
+	_ = s.list.Add(s.p1)
+	_ = s.list.Add(s.p3)
+	_ = s.list.Add(s.p5)
 	reflist := NewPackageRefListFromPackageList(s.list)
 
 	c.Check(reflist.Has(s.p1), Equals, true)
@@ -180,21 +191,21 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
 	}
 
 	for _, p := range packages {
-		coll.Update(p)
+		_ = coll.Update(p)
 	}
 
 	listA := NewPackageList()
-	listA.Add(packages[0])
-	listA.Add(packages[1])
-	listA.Add(packages[2])
-	listA.Add(packages[3])
-	listA.Add(packages[6])
+	_ = listA.Add(packages[0])
+	_ = listA.Add(packages[1])
+	_ = listA.Add(packages[2])
+	_ = listA.Add(packages[3])
+	_ = listA.Add(packages[6])
 
 	listB := NewPackageList()
-	listB.Add(packages[0])
-	listB.Add(packages[2])
-	listB.Add(packages[4])
-	listB.Add(packages[5])
+	_ = listB.Add(packages[0])
+	_ = listB.Add(packages[2])
+	_ = listB.Add(packages[4])
+	_ = listB.Add(packages[5])
 
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
@@ -248,15 +259,15 @@ func (s *PackageRefListSuite) TestDiffCompactsAtEnd(c *C) {
 	}
 
 	for _, p := range packages {
-		coll.Update(p)
+		_ = coll.Update(p)
 	}
 
 	listA := NewPackageList()
-	listA.Add(packages[0])
+	_ = listA.Add(packages[0])
 
 	listB := NewPackageList()
-	listB.Add(packages[1])
-	listB.Add(packages[2])
+	_ = listB.Add(packages[1])
+	_ = listB.Add(packages[2])
 
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
@@ -291,27 +302,27 @@ func (s *PackageRefListSuite) TestMerge(c *C) {
 
 	for _, p := range packages {
 		p.V06Plus = true
-		coll.Update(p)
+		_ = coll.Update(p)
 	}
 
 	listA := NewPackageList()
-	listA.Add(packages[0])
-	listA.Add(packages[1])
-	listA.Add(packages[2])
-	listA.Add(packages[3])
-	listA.Add(packages[7])
+	_ = listA.Add(packages[0])
+	_ = listA.Add(packages[1])
+	_ = listA.Add(packages[2])
+	_ = listA.Add(packages[3])
+	_ = listA.Add(packages[7])
 
 	listB := NewPackageList()
-	listB.Add(packages[0])
-	listB.Add(packages[2])
-	listB.Add(packages[4])
-	listB.Add(packages[5])
-	listB.Add(packages[6])
+	_ = listB.Add(packages[0])
+	_ = listB.Add(packages[2])
+	_ = listB.Add(packages[4])
+	_ = listB.Add(packages[5])
+	_ = listB.Add(packages[6])
 
 	listC := NewPackageList()
-	listC.Add(packages[0])
-	listC.Add(packages[8])
-	listC.Add(packages[9])
+	_ = listC.Add(packages[0])
+	_ = listC.Add(packages[8])
+	_ = listC.Add(packages[9])
 
 	reflistA := NewPackageRefListFromPackageList(listA)
 	reflistB := NewPackageRefListFromPackageList(listB)
@@ -372,14 +383,14 @@ func (s *PackageRefListSuite) TestFilterLatestRefs(c *C) {
 	}
 
 	rl := NewPackageList()
-	rl.Add(packages[0])
-	rl.Add(packages[1])
-	rl.Add(packages[2])
-	rl.Add(packages[3])
-	rl.Add(packages[4])
-	rl.Add(packages[5])
-	rl.Add(packages[6])
-	rl.Add(packages[7])
+	_ = rl.Add(packages[0])
+	_ = rl.Add(packages[1])
+	_ = rl.Add(packages[2])
+	_ = rl.Add(packages[3])
+	_ = rl.Add(packages[4])
+	_ = rl.Add(packages[5])
+	_ = rl.Add(packages[6])
+	_ = rl.Add(packages[7])
 
 	result := NewPackageRefListFromPackageList(rl)
 	result.FilterLatestRefs()