document prometheus API

* enable in dev and test env
* fix api/repos doc

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,6 +4,10 @@ Fixes #
 
 All new code should be covered with tests, documentation should be updated. CI should pass.
 
+Also, to speed up things, if you could kindly "Allow edits and access to secrets by maintainers" in the
+PR settings, as this allows us to rebase the PR on master, fix conflicts, run coverage and help with
+implementing code and tests.
+
 ## Description of the Change
 
 <!--
@@ -14,6 +18,7 @@ Why this change is important?
 
 ## Checklist
 
+- [ ] allow Maintainers to edit PR (rebase, run coverage, help with tests, ...)
 - [ ] unit-test added (if change is algorithm)
 - [ ] functional test added/updated (if change is functional)
 - [ ] man page updated (if applicable)

.github/workflows/ci.yml

@@ -17,8 +17,32 @@ env:
   DEBIAN_FRONTEND: noninteractive
 
 jobs:
+  unit-test:
+    name: "Unit Tests"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+        with:
+          # fetch the whole repo for `git describe` to work
+          fetch-depth: 0
+      - name: "Docker Image"
+        run: |
+          make docker-image
+      - name: "Unit Tests"
+        run: |
+          make docker-unit-tests
+          mkdir -p out/coverage
+          mv unit.out out/coverage/
+      - uses: actions/upload-artifact@v4
+        with:
+          name: unit-tests-coverage
+          path: out/
+
   test:
-    name: "Test (Ubuntu 22.04)"
+    name: "System Test"
     runs-on: ubuntu-22.04
     continue-on-error: false
     timeout-minutes: 30
@@ -30,10 +54,10 @@ jobs:
       GOPROXY: "https://proxy.golang.org"
 
     steps:
-      - name: "Install Packages"
+      - name: "Install Test Packages"
         run: |
           sudo apt-get update
-          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
+          sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
 
       - name: "Checkout Repository"
         uses: actions/checkout@v4
@@ -63,21 +87,10 @@ jobs:
         with:
           directory: ${{ runner.temp }}
 
-      - name: "Run Unit Tests"
-        env:
-          RUN_LONG_TESTS: 'yes'
-          AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
-          AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
-          AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        run: |
-          sudo mkdir -p /srv ; sudo chown runner /srv
-          COVERAGE_DIR=${{ runner.temp }} make test
-
       - name: "Run Benchmark"
         run: |
-          COVERAGE_DIR=${{ runner.temp }} make bench
+          mkdir -p out/coverage
+          COVERAGE_DIR=$PWD/out/coverage make bench
 
       - name: "Run System Tests"
         env:
@@ -89,30 +102,63 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
           sudo mkdir -p /srv ; sudo chown runner /srv
-          COVERAGE_DIR=${{ runner.temp }} make system-test
+          mkdir -p out/coverage
+          COVERAGE_DIR=$PWD/out/coverage make system-test
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: system-tests-coverage
+          path: out/
+
+  coverage:
+    name: "Upload Coverage"
+    runs-on: ubuntu-22.04
+    continue-on-error: false
+    timeout-minutes: 30
+    needs:
+      - unit-test
+      - test
+    steps:
+      - name: "Checkout Repository"
+        uses: actions/checkout@v4
+
+      - name: "Download Unit Test Coverage"
+        uses: actions/download-artifact@v4
+        with:
+          name: unit-tests-coverage
+
+      - name: "Download System Test Coverage"
+        uses: actions/download-artifact@v4
+        with:
+          name: system-tests-coverage
 
       - name: "Merge Code Coverage"
         run: |
-          go install github.com/wadey/gocovmerge@latest
+          # go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
-          ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
+          # ~/go/bin/gocovmerge coverage/*.out > coverage.txt
+          awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
 
       - name: "Upload Code Coverage"
-        uses: codecov/codecov-action@v2
+        if: github.actor != 'dependabot[bot]'
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.txt
+          fail_ci_if_error: true
+
 
   ci-debian-build:
     name: "Build"
-    needs: test
+    needs:
+      - coverage
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
+        name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
         arch: ["amd64", "i386" , "arm64" , "armhf"]
         include:
-          - name: "Debian 13/testing"
+          - name: "Debian 13/trixie"
             suite: trixie
             image: debian:trixie-slim
           - name: "Debian 12/bookworm"
@@ -121,9 +167,6 @@ jobs:
           - name: "Debian 11/bullseye"
             suite: bullseye
             image: debian:bullseye-slim
-          - name: "Debian 10/buster"
-            suite: buster
-            image: debian:buster-slim
           - name: "Ubuntu 24.04"
             suite: noble
             image: ubuntu:24.04
@@ -135,11 +178,12 @@ jobs:
             image: ubuntu:20.04
     container:
       image: ${{ matrix.image }}
+      options: --user root
       env:
         APT_LISTCHANGES_FRONTEND: none
         DEBIAN_FRONTEND: noninteractive
     steps:
-      - name: "Install packages"
+      - name: "Install Build Packages"
         run: |
           apt-get update
           apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
@@ -207,9 +251,27 @@ jobs:
         run: |
           .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
 
+      - name: "Get aptly version"
+        env:
+          FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
+        run: |
+           aptlyver=$(make -s version)
+           echo "Aptly Version: $aptlyver"
+           echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
+        id: releaseversion
+
+      - name: "Upload CI Artifacts"
+        if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags')
+        uses: actions/upload-artifact@v4
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }}
+          path: build/
+          retention-days: 7
+
   ci-binary-build:
     name: "Build"
-    needs: test
+    needs:
+      - coverage
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -267,6 +329,15 @@ jobs:
           path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
           compression-level: 0 # no compression
 
+      - name: "Upload CI Artifacts"
+        uses: actions/upload-artifact@v4
+        if: "!startsWith(github.event.ref, 'refs/tags')"
+        with:
+          name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
+          path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
+          compression-level: 0 # no compression
+          retention-days: 7
+
   gh-release:
     name: "Github Release"
     runs-on: ubuntu-latest

.github/workflows/golangci-lint.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Install and initialize swagger
         run: |
           go install github.com/swaggo/swag/cmd/swag@latest
-          swag init -q --markdownFiles docs
+          swag init -q --propertyStrategy pascalcase --markdownFiles docs
         shell: sh
 
       - name: golangci-lint
@@ -44,7 +44,7 @@ jobs:
           # Require: The version of golangci-lint to use.
           # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
           # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
-          version: v1.54.1
+          version: v1.64.5
 
           # Optional: working directory, useful for monorepos
           # working-directory: somedir

.gitignore

@@ -26,6 +26,8 @@ _testmain.go
 *.test
 
 coverage.txt
+coverage.out
+coverage.html
 
 *.pyc
 

.golangci.yml

@@ -1,16 +1,11 @@
-run:
+version: "2"
-  tests: false
-
-
 linters:
-  disable-all: true
+  settings:
-  enable:
+    staticcheck:
-   - goconst
+      checks:
-   - gofmt
+      - "all"
-   - goimports
+      - "-QF1004"  # could use strings.ReplaceAll instead
-   - govet
+      - "-QF1012"  # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))
-   - ineffassign
+      - "-QF1003"  # could use tagged switch
-   - misspell
+      - "-ST1000"  # at least one file in a package should have a package comment
-   - revive
+      - "-QF1001"  # could apply De Morgan's law
-   - staticcheck
-   - vetshadow

AUTHORS

@@ -68,3 +68,19 @@ List of contributors, in chronological order:
 * Blake Kostner (https://github.com/btkostner)
 * Leigh London (https://github.com/leighlondon)
 * Gordian Schoenherr (https://github.com/schoenherrg)
+* Silke Hofstra (https://github.com/silkeh)
+* Itay Porezky (https://github.com/itayporezky)
+* Alejandro Guijarro Monerris (https://github.com/alguimodd)
+* JupiterRider (https://github.com/JupiterRider)
+* Agustin Henze (https://github.com/agustinhenze)
+* Tobias Assarsson (https://github.com/daedaluz)
+* Yaksh Bariya (https://github.com/thunder-coding)
+* Juan Calderon-Perez (https://github.com/gaby)
+* Ato Araki (https://github.com/atotto)
+* Roman Lebedev (https://github.com/LebedevRI)
+* Brian Witt (https://github.com/bwitt)
+* Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)
+* Zhang Xiao (https://github.com/xzhang1)
+* Tom Nguyen (https://github.com/lecafard)
+* Philip Cramer (https://github.com/PhilipCramer)

Makefile

@@ -2,13 +2,32 @@ GOPATH=$(shell go env GOPATH)
 VERSION=$(shell make -s version)
 PYTHON?=python3
 BINPATH?=$(GOPATH)/bin
-GOLANGCI_LINT_VERSION=v1.54.1  # version supporting go 1.19
+GOLANGCI_LINT_VERSION=v2.0.2  # version supporting go 1.24
 COVERAGE_DIR?=$(shell mktemp -d)
 GOOS=$(shell go env GOHOSTOS)
 GOARCH=$(shell go env GOHOSTARCH)
 
-# Uncomment to update system test gold files
+export PODMAN_USERNS = keep-id
-# CAPTURE := "--capture"
+DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
+
+# Setting TZ for certificates
+export TZ=UTC
+# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
+export TEST_FAKETIME := 2025-01-02 03:04:05
+
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
+else
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
+endif
+
+# export CAPUTRE=1 for regenrating test gold files
+ifeq ($(CAPTURE),1)
+CAPTURE_ARG := --capture
+endif
 
 help:  ## Print this help
 	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -50,7 +69,7 @@ swagger-install:
 	echo "// @version $(VERSION)" >> docs/swagger.conf
 
 azurite-start:
-	azurite -l /tmp/aptly-azurite & \
+	azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
 	echo $$! > ~/.azurite.pid
 
 azurite-stop:
@@ -58,7 +77,7 @@ azurite-stop:
 
 swagger: swagger-install
 	# Generate swagger docs
-	@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+	@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
 
 etcd-install:
 	# Install etcd
@@ -69,9 +88,9 @@ flake8:  ## run flake8 on system test python files
 
 lint: prepare
 	# Install golangci-lint
-	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
 	# Running lint
-	@PATH=$(BINPATH)/:$(PATH) golangci-lint run
+	@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0
 
 
 build: prepare swagger  ## Build aptly
@@ -84,11 +103,11 @@ install:
 	# go install -v
 	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
 
-test: prepare swagger etcd-install  ## Run unit tests
+test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify which tests to run)
 	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
 	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
 	@echo "\e[33m\e[1mRunning go test ...\e[0m"
-	go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
+	faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
 	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
 	@pid=`cat /tmp/etcd.pid`; kill $$pid
 	@rm -f /tmp/aptly-etcd-data/etcd.log
@@ -96,13 +115,13 @@ test: prepare swagger etcd-install  ## Run unit tests
 
 system-test: prepare swagger etcd-install  ## Run system tests
 	# build coverage binary
-	go test -v -coverpkg="./..." -c -tags testruncli
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
 	# Download fixture-db, fixture-pool, etcd.db
 	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
 	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
 	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
 	# Run system tests
-	PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
 
 bench:
 	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -112,7 +131,8 @@ serve: prepare swagger-install  ## Run development server (auto recompiling)
 	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
 	cp debian/aptly.conf ~/.aptly.conf
 	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
-	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
 
 dpkg: prepare swagger  ## Build debian packages
 	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
@@ -162,44 +182,49 @@ binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux gener
 docker-image:  ## Build aptly-dev docker image
 	@docker build -f system/Dockerfile . -t aptly-dev
 
+docker-image-no-cache:  ## Build aptly-dev docker image (no cache)
+	@docker build --no-cache -f system/Dockerfile . -t aptly-dev
+
 docker-build:  ## Build aptly in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
 
 docker-shell:  ## Run aptly and other commands in docker container
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
+	@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
 
 docker-deb:  ## Build debian packages in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
 
-docker-unit-test:  ## Run unit tests in docker container
+docker-unit-tests:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
-		test \
+		test TEST=$(TEST) \
 		azurite-stop
 
 docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
 		azurite-start \
 		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
 		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
 		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
-		system-test TEST=$(TEST) \
+		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
+		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
 		azurite-stop
 
 docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
-	@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
+	@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
 
 docker-lint:  ## Run golangci-lint in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
 
 docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
 
 docker-man:  ## Create man page in docker container
-	@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
 
 mem.png: mem.dat mem.gp
 	gnuplot mem.gp
@@ -216,4 +241,4 @@ clean:  ## remove local build and module cache
 	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
 	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
 
-.PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -63,7 +63,7 @@ Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
 
     deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
 
-Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
+Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
 
 Install aptly packages::
 
@@ -80,7 +80,7 @@ Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
 
     deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
 
-Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
+Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
 
 Note: same gpg key is used as for the Upstream Debian Packages.
 
@@ -111,10 +111,8 @@ With configuration management systems:
 
 -   `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
     (Heavy Water Operations, LLC)
--   `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
+-   `Puppet module <https://github.com/voxpupuli/puppet-aptly>`_ by
-    Government Digital Services
+    Vox Pupuli
--   `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
-    TubeMogul
 -   `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
     Forrest Alvarez and Brian Jackson
 -   `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine

Releasing.md

@@ -12,5 +12,7 @@ git push origin v$version master
 ```
 - run swagger locally (`make docker-serve`)
 - copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
-- releae www.aptly.info
+- add new version to select tag in content/doc/api/swagger.md line 48
+- update version in content/download.md
+- push commit to master
 - create release announcement on https://github.com/aptly-dev/aptly/discussions

api/api.go

@@ -41,7 +41,10 @@ type aptlyVersion struct {
 // @Success 200 {object} aptlyVersion
 // @Router /api/version [get]
 func apiVersion(c *gin.Context) {
-	c.JSON(200, gin.H{"Version": aptly.Version})
+	version := aptlyVersion{
+		Version: aptly.Version,
+	}
+	c.JSON(200, version)
 }
 
 type aptlyStatus struct {
@@ -67,7 +70,8 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) {
 			return
 		}
 
-		c.JSON(200, gin.H{"Status": "Aptly is ready"})
+		status := aptlyStatus{Status: "Aptly is ready"}
+		c.JSON(200, status)
 	}
 }
 
@@ -165,7 +169,7 @@ func runTaskInBackground(name string, resources []string, proc task.Process) (ta
 			return nil, err
 		}
 
-		defer releaseDatabaseConnection()
+		defer func() { _ = releaseDatabaseConnection() }()
 		return proc(out, detail)
 	})
 }
@@ -174,18 +178,18 @@ func truthy(value interface{}) bool {
 	if value == nil {
 		return false
 	}
-	switch value.(type) {
+	switch v := value.(type) {
 	case string:
-		switch strings.ToLower(value.(string)) {
+		switch strings.ToLower(v) {
 		case "n", "no", "f", "false", "0", "off":
 			return false
 		default:
 			return true
 		}
 	case int:
-		return !(value.(int) == 0)
+		return v != 0
 	case bool:
-		return value.(bool)
+		return v
 	}
 	return true
 }
@@ -210,11 +214,11 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 		}
 
 		// wait for task to finish
-		context.TaskList().WaitForTaskByID(task.ID)
+		_, _ = context.TaskList().WaitForTaskByID(task.ID)
 
 		retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
 		err, _ := context.TaskList().GetTaskErrorByID(task.ID)
-		context.TaskList().DeleteTaskByID(task.ID)
+		_, _ = context.TaskList().DeleteTaskByID(task.ID)
 		if err != nil {
 			AbortWithJSONError(c, retValue.Code, err)
 			return
@@ -229,7 +233,7 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
 
 // Common piece of code to show list of packages,
 // with searching & details if requested
-func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb.CollectionFactory) {
+func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) {
 	result := []*deb.Package{}
 
 	list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
@@ -282,11 +286,11 @@ func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb
 	// filter packages by version
 	if c.Request.URL.Query().Get("maximumVersion") == "1" {
 		list.PrepareIndex()
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
 			if err != nil {
 				fmt.Println("filter packages by version, query string parse err: ", err)
-				c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
+				_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
 			} else {
 				tmpList, err := list.Filter(deb.FilterOptions{
 					Queries: []deb.PackageQuery{versionQ},
@@ -294,15 +298,15 @@ func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb
 
 				if err == nil {
 					if tmpList.Len() > 0 {
-						tmpList.ForEach(func(tp *deb.Package) error {
+						_ = tmpList.ForEach(func(tp *deb.Package) error {
 							list.Remove(tp)
 							return nil
 						})
-						list.Add(p)
+						_ = list.Add(p)
 					}
 				} else {
 					fmt.Println("filter packages by version, filter err: ", err)
-					c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
+					_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
 				}
 			}
 
@@ -311,7 +315,7 @@ func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -322,7 +326,7 @@ func showPackages(c *gin.Context, reflist deb.AnyRefList, collectionFactory *deb
 	}
 }
 
-func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
+func AbortWithJSONError(c *gin.Context, code int, err error) {
 	c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
-	return c.AbortWithError(code, err)
+	_ = c.AbortWithError(code, err)
 }

api/api_test.go

@@ -24,14 +24,14 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ApiSuite struct {
+type APISuite struct {
 	context    *ctx.AptlyContext
 	flags      *flag.FlagSet
 	configFile *os.File
 	router     http.Handler
 }
 
-var _ = Suite(&ApiSuite{})
+var _ = Suite(&APISuite{})
 
 func createTestConfig() *os.File {
 	file, err := os.CreateTemp("", "aptly")
@@ -45,11 +45,11 @@ func createTestConfig() *os.File {
 	if err != nil {
 		return nil
 	}
-	file.Write(jsonString)
+	_, _ = file.Write(jsonString)
 	return file
 }
 
-func (s *ApiSuite) setupContext() error {
+func (s *APISuite) setupContext() error {
 	aptly.Version = "testVersion"
 	file := createTestConfig()
 	if nil == file {
@@ -75,23 +75,23 @@ func (s *ApiSuite) setupContext() error {
 	return nil
 }
 
-func (s *ApiSuite) SetUpSuite(c *C) {
+func (s *APISuite) SetUpSuite(c *C) {
 	err := s.setupContext()
 	c.Assert(err, IsNil)
 }
 
-func (s *ApiSuite) TearDownSuite(c *C) {
+func (s *APISuite) TearDownSuite(c *C) {
-	os.Remove(s.configFile.Name())
+	_ = os.Remove(s.configFile.Name())
 	s.context.Shutdown()
 }
 
-func (s *ApiSuite) SetUpTest(c *C) {
+func (s *APISuite) SetUpTest(c *C) {
 }
 
-func (s *ApiSuite) TearDownTest(c *C) {
+func (s *APISuite) TearDownTest(c *C) {
 }
 
-func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
+func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
 	w := httptest.NewRecorder()
 	req, err := http.NewRequest(method, url, body)
 	if err != nil {
@@ -102,32 +102,32 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
 	return w, nil
 }
 
-func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
+func (s *APISuite) TestGinRunsInReleaseMode(c *C) {
 	c.Check(gin.Mode(), Equals, gin.ReleaseMode)
 }
 
-func (s *ApiSuite) TestGetVersion(c *C) {
+func (s *APISuite) TestGetVersion(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/version", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
 }
 
-func (s *ApiSuite) TestGetReadiness(c *C) {
+func (s *APISuite) TestGetReadiness(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/ready", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
 }
 
-func (s *ApiSuite) TestGetHealthiness(c *C) {
+func (s *APISuite) TestGetHealthiness(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/healthy", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
 }
 
-func (s *ApiSuite) TestGetMetrics(c *C) {
+func (s *APISuite) TestGetMetrics(c *C) {
 	response, err := s.HTTPRequest("GET", "/api/metrics", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
@@ -141,7 +141,7 @@ func (s *ApiSuite) TestGetMetrics(c *C) {
 	c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
 }
 
-func (s *ApiSuite) TestRepoCreate(c *C) {
+func (s *APISuite) TestRepoCreate(c *C) {
 	body, err := json.Marshal(gin.H{
 		"Name": "dummy",
 	})
@@ -150,7 +150,7 @@ func (s *ApiSuite) TestRepoCreate(c *C) {
 	c.Assert(err, IsNil)
 }
 
-func (s *ApiSuite) TestTruthy(c *C) {
+func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy("no"), Equals, false)
 	c.Check(truthy("n"), Equals, false)
 	c.Check(truthy("off"), Equals, false)

api/db.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 
 	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/task"
 	"github.com/aptly-dev/aptly/utils"
@@ -22,29 +21,25 @@ import (
 // @Success 200 {object} string "Output"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/db/cleanup [post]
-func apiDbCleanup(c *gin.Context) {
+func apiDBCleanup(c *gin.Context) {
 	resources := []string{string(task.AllResourcesKey)}
 	maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
 		var err error
 
 		collectionFactory := context.NewCollectionFactory()
 
-		// collect information about referenced packages and their reflist buckets...
+		// collect information about referenced packages...
-		existingPackageRefs := deb.NewSplitRefList()
+		existingPackageRefs := deb.NewPackageRefList()
-		existingBuckets := deb.NewRefListDigestSet()
-
-		reflistMigration := collectionFactory.RefListCollection().NewMigration()
 
 		out.Printf("Loading mirrors, local repos, snapshots and published repos...")
 		err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-			sl := deb.NewSplitRefList()
+			e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
+			if e != nil {
-			if e != nil && e != database.ErrNotFound {
 				return e
 			}
-
+			if repo.RefList() != nil {
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
-			existingBuckets.AddAllInRefList(sl)
+			}
 
 			return nil
 		})
@@ -53,14 +48,14 @@ func apiDbCleanup(c *gin.Context) {
 		}
 
 		err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-			sl := deb.NewSplitRefList()
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
+			if e != nil {
-			if e != nil && e != database.ErrNotFound {
 				return e
 			}
 
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+			if repo.RefList() != nil {
-			existingBuckets.AddAllInRefList(sl)
+				existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
+			}
 
 			return nil
 		})
@@ -69,14 +64,12 @@ func apiDbCleanup(c *gin.Context) {
 		}
 
 		err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-			sl := deb.NewSplitRefList()
+			e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, snapshot.RefKey(), reflistMigration)
 			if e != nil {
 				return e
 			}
 
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+			existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
-			existingBuckets.AddAllInRefList(sl)
 
 			return nil
 		})
@@ -88,16 +81,13 @@ func apiDbCleanup(c *gin.Context) {
 			if published.SourceKind != deb.SourceLocalRepo {
 				return nil
 			}
+			e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+			if e != nil {
+				return e
+			}
 
 			for _, component := range published.Components() {
-				sl := deb.NewSplitRefList()
+				existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
-				e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, published.RefKey(component), reflistMigration)
-				if e != nil {
-					return e
-				}
-
-				existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-				existingBuckets.AddAllInRefList(sl)
 			}
 			return nil
 		})
@@ -105,20 +95,11 @@ func apiDbCleanup(c *gin.Context) {
 			return nil, err
 		}
 
-		err = reflistMigration.Flush()
-		if err != nil {
-			return nil, err
-		}
-		if stats := reflistMigration.Stats(); stats.Reflists > 0 {
-			out.Printf("Split %d reflist(s) into %d bucket(s) (%d segment(s))",
-				stats.Reflists, stats.Buckets, stats.Segments)
-		}
-
 		// ... and compare it to the list of all packages
 		out.Printf("Loading list of all packages...")
 		allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
-		toDelete := allPackageRefs.Subtract(existingPackageRefs.Flatten())
+		toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 		// delete packages that are no longer referenced
 		out.Printf("Deleting unreferenced packages (%d)...", toDelete.Len())
@@ -128,8 +109,8 @@ func apiDbCleanup(c *gin.Context) {
 
 		if toDelete.Len() > 0 {
 			batch := db.CreateBatch()
-			toDelete.ForEach(func(ref []byte) error {
+			_ = toDelete.ForEach(func(ref []byte) error {
-				collectionFactory.PackageCollection().DeleteByKey(ref, batch)
+				_ = collectionFactory.PackageCollection().DeleteByKey(ref, batch)
 				return nil
 			})
 
@@ -139,28 +120,6 @@ func apiDbCleanup(c *gin.Context) {
 			}
 		}
 
-		bucketsToDelete, err := collectionFactory.RefListCollection().AllBucketDigests()
-		if err != nil {
-			return nil, err
-		}
-
-		bucketsToDelete.RemoveAll(existingBuckets)
-
-		out.Printf("Deleting unreferenced reflist buckets (%d)...", bucketsToDelete.Len())
-		if bucketsToDelete.Len() > 0 {
-			batch := db.CreateBatch()
-			err := bucketsToDelete.ForEach(func(digest []byte) error {
-				return collectionFactory.RefListCollection().UnsafeDropBucket(digest, batch)
-			})
-			if err != nil {
-				return nil, err
-			}
-
-			if err := batch.Write(); err != nil {
-				return nil, err
-			}
-		}
-
 		// now, build a list of files that should be present in Repository (package pool)
 		out.Printf("Building list of files referenced by packages...")
 		referencedFiles := make([]string, 0, existingPackageRefs.Len())

api/files.go

@@ -13,6 +13,10 @@ import (
 	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 func verifyPath(path string) bool {
 	path = filepath.Clean(path)
 	for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -114,34 +118,69 @@ func apiFilesUpload(c *gin.Context) {
 	}
 
 	stored := []string{}
+	openFiles := []*os.File{}
 
+	// Write all files first
 	for _, files := range c.Request.MultipartForm.File {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer src.Close()
 
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
+				_ = src.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer dst.Close()
 
 			_, err = io.Copy(dst, src)
+			_ = src.Close()
 			if err != nil {
+				_ = dst.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
 
+			// Keep file open for batch sync
+			openFiles = append(openFiles, dst)
 			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
 		}
 	}
 
+	// Sync all files at once to catch ENOSPC errors
+	for i, dst := range openFiles {
+		err := syncFile(dst)
+		if err != nil {
+			// Close all files
+			for _, f := range openFiles {
+				_ = f.Close()
+			}
+			AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
+			return
+		}
+	}
+
+	// Close all files
+	for _, dst := range openFiles {
+		_ = dst.Close()
+	}
+
 	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
 }
@@ -170,7 +209,11 @@ func apiFilesListFiles(c *gin.Context) {
 	listLock := &sync.Mutex{}
 	root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
 
-	err := walker.Walk(root, func(path string, _ os.FileInfo) error {
+	err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
 		if path == root {
 			return nil
 		}

api/files_test.go

@@ -0,0 +1,476 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+type FilesUploadDiskFullSuite struct {
+	aptlyContext *ctx.AptlyContext
+	flags        *flag.FlagSet
+	configFile   *os.File
+	router       http.Handler
+}
+
+var _ = Suite(&FilesUploadDiskFullSuite{})
+
+func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
+	aptly.Version = "testVersion"
+
+	file, err := os.CreateTemp("", "aptly")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+		"rootDir":       c.MkDir(),
+	})
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+	_ = file.Close()
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	aptlyContext, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.aptlyContext = aptlyContext
+	s.router = Router(aptlyContext)
+	context = aptlyContext
+}
+
+func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.aptlyContext != nil {
+		s.aptlyContext.Shutdown()
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
+	testContent := []byte("test file content for upload")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "testfile.txt")
+	c.Assert(err, IsNil)
+
+	_, err = part.Write(testContent)
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
+	testContent := bytes.Repeat([]byte("A"), 10000)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "largefile.bin")
+	c.Assert(err, IsNil)
+
+	_, err = io.Copy(part, bytes.NewReader(testContent))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir2", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(len(content), Equals, len(testContent))
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
+	testFiles := map[string][]byte{
+		"file1.txt": []byte("content of file 1"),
+		"file2.txt": bytes.Repeat([]byte("B"), 5000),
+		"file3.deb": []byte("debian package content"),
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	for filename, content := range testFiles {
+		part, err := writer.CreateFormFile("file", filename)
+		c.Assert(err, IsNil)
+		_, err = part.Write(content)
+		c.Assert(err, IsNil)
+	}
+
+	err := writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/multitest", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
+	for filename, expectedContent := range testFiles {
+		uploadedFile := filepath.Join(uploadDir, filename)
+		content, err := os.ReadFile(uploadedFile)
+		c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
+		c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
+	oldSyncFile := syncFile
+	syncFile = func(f *os.File) error {
+		if filepath.Base(f.Name()) == "syncfail.txt" {
+			return syscall.ENOSPC
+		}
+		return nil
+	}
+	defer func() { syncFile = oldSyncFile }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part1, err := writer.CreateFormFile("file", "ok.txt")
+	c.Assert(err, IsNil)
+	_, err = part1.Write([]byte("ok"))
+	c.Assert(err, IsNil)
+
+	part2, err := writer.CreateFormFile("file", "syncfail.txt")
+	c.Assert(err, IsNil)
+	_, err = part2.Write([]byte("will fail on sync"))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+	c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
+	c.Check(verifyPath("a/b/c"), Equals, true)
+	c.Check(verifyPath("../x"), Equals, false)
+	c.Check(verifyPath("./x"), Equals, true)
+	c.Check(verifyPath(".."), Equals, false)
+	c.Check(verifyPath("."), Equals, false)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
+	uploadRoot := s.aptlyContext.UploadPath()
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "d1"), Equals, true)
+	c.Check(strings.Contains(body, "d2"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
+	req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 404)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files/dir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "a.txt"), Equals, true)
+	c.Check(strings.Contains(body, "b.txt"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(base)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(filepath.Join(base, "a.txt"))
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
+	req, err := http.NewRequest("DELETE", "/api/files/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.Chmod(base, 0), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	req, err := http.NewRequest("GET", "/api/files/noperms", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	subdir := filepath.Join(base, "subdir")
+	c.Assert(os.MkdirAll(subdir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
+	req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/..", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
+	// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
+	form := &multipart.Form{
+		File: map[string][]*multipart.FileHeader{
+			"file": {{Filename: "broken.bin"}},
+		},
+	}
+
+	req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
+	c.Assert(err, IsNil)
+	req.MultipartForm = form
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.Chmod(base, 0555), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/readonly", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
+	parent := s.aptlyContext.UploadPath()
+	base := filepath.Join(parent, "cantremove")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	c.Assert(os.Chmod(parent, 0555), IsNil)
+	defer func() { _ = os.Chmod(parent, 0777) }()
+
+	req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}

api/gpg.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"os/exec"
@@ -12,27 +13,60 @@ import (
 	"github.com/gin-gonic/gin"
 )
 
+type gpgKeyInfo struct {
+	// 16-character key ID (short form)
+	KeyID string `json:"KeyID"         example:"8B48AD6246925553"`
+	// Full fingerprint
+	Fingerprint string `json:"Fingerprint"   example:"D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0"`
+	// Key validity (u=unknown, f=fulltrust, m=marginal, n=never)
+	Validity string `json:"Validity"      example:"u"`
+	// User ID(s) associated with this key
+	UserIDs []string `json:"UserIDs"       example:"John Doe <john@example.com>"`
+	// Creation date (Unix timestamp format from gpg)
+	CreatedAt string `json:"CreatedAt"     example:"2023-01-15"`
+}
+
+type gpgKeyListResponse struct {
+	Keys []gpgKeyInfo `json:"Keys"`
+}
+
 type gpgAddKeyParams struct {
-	// Keyserver, when downloading GpgKeyIDs
-	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
-	// GpgKeyIDs to download from Keyserver, comma separated list
-	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500,8B48AD6246925553"`
-	// Armored gpg public ket, instead of downloading from keyserver
-	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
 	// Keyring for adding the keys (default: trustedkeys.gpg)
 	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Add ASCII armored gpg public key, do not download from keyserver
+	GpgKeyArmor string `json:"GpgKeyArmor"     example:""`
+
+	// Keyserver to download keys provided in `GpgKeyID`
+	Keyserver string `json:"Keyserver"       example:"hkp://keyserver.ubuntu.com:80"`
+	// Keys do download from `Keyserver`, separated by space
+	GpgKeyID string `json:"GpgKeyID"        example:"EF0F382A1A7B6500 8B48AD6246925553"`
+}
+
+type gpgDeleteKeyParams struct {
+	// Keyring to delete keys from (default: trustedkeys.gpg)
+	Keyring string `json:"Keyring"         example:"trustedkeys.gpg"`
+
+	// Key ID or fingerprint to delete
+	GpgKeyID string `json:"GpgKeyID"        example:"8B48AD6246925553"`
 }
 
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
 // @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description
+// @Description Keys can be added in two ways:
+// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
+// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
+// @Description
 // @Tags Mirrors
+// @Consume  json
+// @Param request body gpgAddKeyParams true "Parameters"
 // @Produce json
 // @Success 200 {object} string "OK"
 // @Failure 400 {object} Error "Bad Request"
-// @Failure 404 {object} Error "Not Found"
+// @Router /api/gpg/key [post]
-// @Router /api/gpg [post]
 func apiGPGAddKey(c *gin.Context) {
 	b := gpgAddKeyParams{}
 	if c.Bind(&b) != nil {
@@ -60,7 +94,7 @@ func apiGPGAddKey(c *gin.Context) {
 			AbortWithJSONError(c, 400, err)
 			return
 		}
-		defer os.RemoveAll(tempdir)
+		defer func() { _ = os.RemoveAll(tempdir) }()
 
 		keypath := filepath.Join(tempdir, "key")
 		keyfile, e := os.Create(keypath)
@@ -100,3 +134,178 @@ func apiGPGAddKey(c *gin.Context) {
 
 	c.JSON(200, string(out))
 }
+
+// @Summary List GPG Keys
+// @Description **Lists all GPG keys in aptly keyring**
+// @Description
+// @Description Returns all public keys currently installed in the aptly GPG keyring.
+// @Description
+// @Tags Mirrors
+// @Param keyring query string false "Keyring file to list keys from (default: trustedkeys.gpg)" example(trustedkeys.gpg)
+// @Produce json
+// @Success 200 {object} gpgKeyListResponse "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/gpg/keys [get]
+func apiGPGListKeys(c *gin.Context) {
+	keyring := c.DefaultQuery("keyring", "trustedkeys.gpg")
+	keyring = utils.SanitizePath(keyring)
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	args := []string{
+		"--no-default-keyring",
+		"--with-colons",
+		"--keyring", keyring,
+		"--list-keys",
+	}
+
+	cmd := exec.Command(gpg, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("failed to list keys: %s", string(out)))
+		return
+	}
+
+	keys := parseGPGOutput(string(out))
+	c.JSON(200, gpgKeyListResponse{Keys: keys})
+}
+
+// @Summary Delete GPG Key
+// @Description **Deletes a GPG key from aptly keyring**
+// @Description
+// @Description Removes a public key from the aptly GPG keyring. This is useful for removing
+// @Description compromised keys or cleaning up obsolete keys.
+// @Description
+// @Tags Mirrors
+// @Consume  json
+// @Param request body gpgDeleteKeyParams true "Parameters"
+// @Produce json
+// @Success 200 {object} string "OK"
+// @Failure 400 {object} Error "Bad Request"
+// @Router /api/gpg/key [delete]
+func apiGPGDeleteKey(c *gin.Context) {
+	b := gpgDeleteKeyParams{}
+	if c.Bind(&b) != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("invalid request body"))
+		return
+	}
+
+	if len(strings.TrimSpace(b.GpgKeyID)) == 0 {
+		AbortWithJSONError(c, 400, fmt.Errorf("GpgKeyID is required"))
+		return
+	}
+
+	b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
+	// b.Keyring can be an absolute path
+
+	finder := pgp.GPGDefaultFinder()
+	gpg, _, err := finder.FindGPG()
+	if err != nil {
+		AbortWithJSONError(c, 400, err)
+		return
+	}
+
+	args := []string{
+		"--no-default-keyring",
+		"--allow-non-selfsigned-uid",
+		"--batch",
+		"--yes",
+	}
+
+	keyring := "trustedkeys.gpg"
+	if len(b.Keyring) > 0 {
+		keyring = b.Keyring
+	}
+
+	args = append(args, "--keyring", keyring)
+	args = append(args, "--delete-keys", b.GpgKeyID)
+
+	cmd := exec.Command(gpg, args...)
+	fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		AbortWithJSONError(c, 400, fmt.Errorf("failed to delete key: %s", string(out)))
+		return
+	}
+
+	c.JSON(200, string(out))
+}
+
+// parseGPGOutput parses the output of `gpg --with-colons --list-keys`
+// and returns a structured list of keys
+func parseGPGOutput(output string) []gpgKeyInfo {
+	var keys []gpgKeyInfo
+	var currentKey *gpgKeyInfo
+
+	scanner := bufio.NewScanner(strings.NewReader(output))
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+
+		parts := strings.Split(line, ":")
+		if len(parts) < 10 {
+			continue
+		}
+
+		recordType := parts[0]
+
+		// pub: public key record
+		if recordType == "pub" {
+			// Save previous key if it exists
+			if currentKey != nil && currentKey.KeyID != "" {
+				keys = append(keys, *currentKey)
+			}
+
+			// Create new key entry
+			// Format: pub:trust:length:algo:keyid:created:expires:uidhash:...
+			keyID := parts[4]
+			if len(keyID) >= 16 {
+				keyID = keyID[len(keyID)-16:] // Last 16 chars = short key ID
+			}
+			validity := parts[1]
+			createdAt := parts[5]
+
+			currentKey = &gpgKeyInfo{
+				KeyID:      keyID,
+				Validity:   validity,
+				CreatedAt:  createdAt,
+				UserIDs:    []string{},
+				Fingerprint: "",
+			}
+		}
+
+		// uid: user ID record
+		if recordType == "uid" && currentKey != nil {
+			// Format: uid:trust:created:expires:keyid:uidhash:uidtype:validity:userID:...
+			if len(parts) >= 10 {
+				userID := parts[9]
+				if userID != "" {
+					currentKey.UserIDs = append(currentKey.UserIDs, userID)
+				}
+			}
+		}
+
+		// fpr: fingerprint record
+		if recordType == "fpr" && currentKey != nil {
+			// Format: fpr:::::::::fingerprint:
+			if len(parts) >= 10 {
+				fingerprint := parts[9]
+				currentKey.Fingerprint = fingerprint
+			}
+		}
+	}
+
+	// Don't forget the last key
+	if currentKey != nil && currentKey.KeyID != "" {
+		keys = append(keys, *currentKey)
+	}
+
+	return keys
+}

api/gpg_test.go

@@ -0,0 +1,451 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"strings"
+
+	. "gopkg.in/check.v1"
+)
+
+type GPGSuite struct {
+	APISuite
+}
+
+var _ = Suite(&GPGSuite{})
+
+func (s *GPGSuite) withFakeGPG(c *C, scriptBody string, test func(scriptPath string)) {
+	tempDir, err := os.MkdirTemp("", "aptly-fake-gpg")
+	c.Assert(err, IsNil)
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	scriptPath := filepath.Join(tempDir, "gpg")
+	err = os.WriteFile(scriptPath, []byte(scriptBody), 0o755)
+	c.Assert(err, IsNil)
+
+	oldPath := os.Getenv("PATH")
+	err = os.Setenv("PATH", tempDir+string(os.PathListSeparator)+oldPath)
+	c.Assert(err, IsNil)
+	defer func() { _ = os.Setenv("PATH", oldPath) }()
+
+	test(scriptPath)
+}
+
+func (s *GPGSuite) fakeGPGScript(c *C, listOutput string, deleteOutput string, deleteError string) string {
+	return "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then\n" +
+		"  echo 'gpg (GnuPG) 2.2.27'\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"args=\"$*\"\n" +
+		"if printf '%s' \"$args\" | grep -q -- '--list-keys'; then\n" +
+		"  cat <<'EOF'\n" + listOutput + "\nEOF\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"if printf '%s' \"$args\" | grep -q -- '--delete-keys'; then\n" +
+		"  if [ -n \"" + strings.ReplaceAll(deleteError, "\n", "") + "\" ]; then\n" +
+		"    echo '" + strings.ReplaceAll(deleteError, "'", "'\\''") + "'\n" +
+		"    exit 1\n" +
+		"  fi\n" +
+		"  cat <<'EOF'\n" + deleteOutput + "\nEOF\n" +
+		"  exit 0\n" +
+		"fi\n" +
+		"echo 'unexpected invocation' >&2\n" +
+		"exit 1\n"
+}
+
+// TestParseGPGOutputEmpty tests parsing of empty GPG output
+func (s *GPGSuite) TestParseGPGOutputEmpty(c *C) {
+	output := ""
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 0)
+}
+
+// TestParseGPGOutputSingleKeyMinimal tests parsing a single key with minimal fields
+func (s *GPGSuite) TestParseGPGOutputSingleKeyMinimal(c *C) {
+	// Minimal valid GPG output with one key
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.KeyID, Equals, "8B48AD6246925553")
+	c.Check(key.Validity, Equals, "u")
+	c.Check(key.CreatedAt, Equals, "1611864000")
+	c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+	c.Check(key.UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
+}
+
+// TestParseGPGOutputMultipleKeys tests parsing multiple keys
+func (s *GPGSuite) TestParseGPGOutputMultipleKeys(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+pub:f:2048:1:A1B2C3D4E5F67890:1580592000:1612128000:uidhash:::scESC:::::::23::0:
+uid:f::::1580592000::0987654321::Jane Smith <jane@example.com>::::::::::0:
+fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 2)
+
+	// First key
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+	c.Check(keys[0].Validity, Equals, "u")
+	c.Check(keys[0].UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
+
+	// Second key
+	c.Check(keys[1].KeyID, Equals, "A1B2C3D4E5F67890")
+	c.Check(keys[1].Validity, Equals, "f")
+	c.Check(keys[1].UserIDs, DeepEquals, []string{"Jane Smith <jane@example.com>"})
+}
+
+// TestParseGPGOutputMultipleUIDs tests a key with multiple user IDs
+func (s *GPGSuite) TestParseGPGOutputMultipleUIDs(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+uid:u::::1611864000::1234567891::John Doe <john.doe@company.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.UserIDs, HasLen, 2)
+	c.Check(key.UserIDs, DeepEquals, []string{
+		"John Doe <john@example.com>",
+		"John Doe <john.doe@company.com>",
+	})
+}
+
+// TestParseGPGOutputMalformedLines tests that malformed lines are skipped
+func (s *GPGSuite) TestParseGPGOutputMalformedLines(c *C) {
+	// Mix of valid and invalid lines (too few fields)
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+invalid:line:with:only:three:fields
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+}
+
+// TestParseGPGOutputEmptyLines tests that empty lines are skipped
+func (s *GPGSuite) TestParseGPGOutputEmptyLines(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+}
+
+// TestParseGPGOutputKeyWithoutUID tests a public key without user ID
+func (s *GPGSuite) TestParseGPGOutputKeyWithoutUID(c *C) {
+	// Key without uid record (should still be included)
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+
+	key := keys[0]
+	c.Check(key.KeyID, Equals, "8B48AD6246925553")
+	c.Check(key.UserIDs, HasLen, 0)
+	c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+}
+
+// TestParseGPGOutputVariousValidity tests different validity values
+func (s *GPGSuite) TestParseGPGOutputVariousValidity(c *C) {
+	output := `pub:u:4096:1:KEY1111111111111:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::Key1::::::::::0:
+fpr:::::::::1111111111111111111111111111111111111111:
+pub:f:4096:1:KEY2222222222222:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:f::::1611864000::1234567891::Key2::::::::::0:
+fpr:::::::::2222222222222222222222222222222222222222:
+pub:m:4096:1:KEY3333333333333:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:m::::1611864000::1234567892::Key3::::::::::0:
+fpr:::::::::3333333333333333333333333333333333333333:
+pub:n:4096:1:KEY4444444444444:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:n::::1611864000::1234567893::Key4::::::::::0:
+fpr:::::::::4444444444444444444444444444444444444444:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 4)
+
+	validities := []string{"u", "f", "m", "n"}
+	for i, validity := range validities {
+		c.Check(keys[i].Validity, Equals, validity)
+	}
+}
+
+// TestParseGPGOutputShortKeyID tests that key IDs are shortened to 16 chars
+func (s *GPGSuite) TestParseGPGOutputShortKeyID(c *C) {
+	// 40-character key ID that should be shortened to last 16 chars
+	longKeyID := "0123456789ABCDEF0123456789ABCDEF8B48AD62"
+	output := `pub:u:4096:1:` + longKeyID + `:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should extract the last 16 characters: 89ABCDEF8B48AD62
+	c.Check(keys[0].KeyID, Equals, "89ABCDEF8B48AD62")
+}
+
+// TestParseGPGOutputSpecialCharactersInUID tests user IDs with special characters
+func (s *GPGSuite) TestParseGPGOutputSpecialCharactersInUID(c *C) {
+	// UID with Unicode characters and special formatting
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::J\xc3\xb6hn D\xc3\xb6\xc3\xa9 (D\xc3\xbcss) <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should preserve the encoded special characters
+	c.Check(keys[0].UserIDs, HasLen, 1)
+}
+
+// TestAPIGPGListKeysDefaultKeyring tests the HTTP endpoint with default keyring
+func (s *GPGSuite) TestAPIGPGListKeysDefaultKeyring(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, `pub:u:4096:1:8B48AD6246925553:1611864000:::::
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`, "", ""), func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		var result gpgKeyListResponse
+		err = json.NewDecoder(response.Body).Decode(&result)
+		c.Assert(err, IsNil)
+		c.Check(result.Keys, HasLen, 1)
+		c.Check(result.Keys[0].KeyID, Equals, "8B48AD6246925553")
+	})
+}
+
+// TestAPIGPGListKeysWithKeyringParam tests the HTTP endpoint with custom keyring parameter
+func (s *GPGSuite) TestAPIGPGListKeysWithKeyringParam(c *C) {
+	argFile, err := os.CreateTemp("", "aptly-gpg-args")
+	c.Assert(err, IsNil)
+	_ = argFile.Close()
+	defer func() { _ = os.Remove(argFile.Name()) }()
+
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
+		"cat <<'EOF'\n" +
+		"pub:u:4096:1:8B48AD6246925553:1611864000:::::\n" +
+		"fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:\n" +
+		"EOF\n" +
+		"exit 0\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		response, reqErr := s.HTTPRequest("GET", "/api/gpg/keys?keyring=/custom.gpg", nil)
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		argBytes, readErr := os.ReadFile(argFile.Name())
+		c.Assert(readErr, IsNil)
+		c.Check(string(argBytes), Matches, `(?s).*--keyring\ncustom\.gpg\n.*`)
+	})
+}
+
+// TestAPIGPGListKeysResponseFormat tests that the response has the correct structure
+func (s *GPGSuite) TestAPIGPGListKeysResponseFormat(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, `pub:f:4096:1:A1B2C3D4E5F67890:1611864000:::::
+uid:f::::1611864000::1234567890::Jane Smith <jane@example.com>::::::::::0:
+fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`, "", ""), func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 200)
+
+		var result gpgKeyListResponse
+		err = json.NewDecoder(response.Body).Decode(&result)
+		c.Assert(err, IsNil)
+		c.Assert(result.Keys, HasLen, 1)
+		c.Check(result.Keys[0].KeyID, Equals, "A1B2C3D4E5F67890")
+		c.Check(result.Keys[0].Validity, Equals, "f")
+		c.Check(result.Keys[0].CreatedAt, Equals, "1611864000")
+	})
+}
+
+// TestParseGPGOutputEdgeCaseUIDWithoutFields tests UID record with missing fields
+func (s *GPGSuite) TestParseGPGOutputEdgeCaseUIDWithoutFields(c *C) {
+	// UID record with fewer than 10 fields
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should not have user ID since it's in field 9 and this record is too short
+	c.Check(keys[0].UserIDs, HasLen, 0)
+}
+
+// TestParseGPGOutputFingerprintWithoutCurrentKey tests FPR record appearing before any PUB
+func (s *GPGSuite) TestParseGPGOutputFingerprintWithoutCurrentKey(c *C) {
+	// FPR record without a preceding PUB (should be ignored)
+	output := `fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should only have one key with the correct fingerprint
+	c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+}
+
+// TestParseGPGOutputComplexRealWorldExample tests real-world-like GPG output
+func (s *GPGSuite) TestParseGPGOutputComplexRealWorldExample(c *C) {
+	// Real-world GPG output with multiple keys, UIDs, and other record types (sig, sub)
+	// Note: sub and sig records are skipped as we only care about pub/uid/fpr
+	realWorldOutput := `tru::1:1611864000:0:3:1:5
+pub:u:4096:1:8B48AD6246925553:1611864000:2023-01-15T00:00:00:::::scESC:::::::23::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
+uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
+uid:u::::1611864100::1234567891::John Doe <john@work.com>::::::::::0:
+pub:f:2048:1:1234567890123456:1580592000:2022-12-31T00:00:00::u:::scESC:::::::23::0:
+fpr:::::::::F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9:
+uid:f::::1580592000::0987654321::Maintainer Key <maint@example.com>::::::::::0:`
+
+	keys := parseGPGOutput(realWorldOutput)
+	c.Check(keys, HasLen, 2)
+
+	// First key should have 2 UIDs
+	c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
+	c.Check(keys[0].UserIDs, HasLen, 2)
+	c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
+
+	// Second key should have 1 UID
+	c.Check(keys[1].KeyID, Equals, "1234567890123456")
+	c.Check(keys[1].UserIDs, HasLen, 1)
+	c.Check(keys[1].Fingerprint, Equals, "F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9")
+}
+
+// TestParseGPGOutputConsecutiveEmptyUIDs tests handling of consecutive empty user ID fields
+func (s *GPGSuite) TestParseGPGOutputConsecutiveEmptyUIDs(c *C) {
+	output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
+uid:u::::1611864000::1234567890:::::::::::0:
+uid:u::::1611864000::1234567891::John Doe <john@example.com>::::::::::0:
+fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
+
+	keys := parseGPGOutput(output)
+	c.Check(keys, HasLen, 1)
+	// Should skip empty UID but include the non-empty one
+	c.Check(keys[0].UserIDs, HasLen, 1)
+	c.Check(keys[0].UserIDs[0], Equals, "John Doe <john@example.com>")
+}
+
+// TestGPGDeleteKeyParamsValidation tests gpgDeleteKeyParams validation
+func (s *GPGSuite) TestGPGDeleteKeyParamsValidation(c *C) {
+	// This is a unit test that validates parameter structure (no HTTP needed)
+	params := gpgDeleteKeyParams{
+		Keyring: "custom.gpg",
+		GpgKeyID: "8B48AD6246925553",
+	}
+	c.Check(params.Keyring, Equals, "custom.gpg")
+	c.Check(params.GpgKeyID, Equals, "8B48AD6246925553")
+}
+
+// TestAPIGPGDeleteKeyMissingKeyID tests delete with missing key ID parameter
+func (s *GPGSuite) TestAPIGPGDeleteKeyMissingKeyID(c *C) {
+	body, err := json.Marshal(map[string]string{
+		"Keyring": "trustedkeys.gpg",
+		// GpgKeyID is missing
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+}
+
+// TestAPIGPGDeleteKeyInvalidJSON tests delete with invalid JSON request
+func (s *GPGSuite) TestAPIGPGDeleteKeyInvalidJSON(c *C) {
+	response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader([]byte("invalid json")))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+}
+
+// TestAPIGPGDeleteKeySuccess tests successful key deletion
+func (s *GPGSuite) TestAPIGPGDeleteKeySuccess(c *C) {
+	argFile, err := os.CreateTemp("", "aptly-gpg-delete-args")
+	c.Assert(err, IsNil)
+	_ = argFile.Close()
+	defer func() { _ = os.Remove(argFile.Name()) }()
+
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--delete-keys'; then\n" +
+		"echo 'deleted'\n" +
+		"exit 0\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		body, marshalErr := json.Marshal(gpgDeleteKeyParams{
+			Keyring: "/trustedkeys.gpg",
+			GpgKeyID: "8B48AD6246925553",
+		})
+		c.Assert(marshalErr, IsNil)
+
+		response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 200)
+		c.Check(response.Body.String(), Matches, `"deleted\\n"`)
+
+		argBytes, readErr := os.ReadFile(argFile.Name())
+		c.Assert(readErr, IsNil)
+		argText := string(argBytes)
+		c.Check(argText, Matches, `(?s).*--batch\n--yes\n.*`)
+		c.Check(argText, Matches, `(?s).*--keyring\n/trustedkeys\.gpg\n.*`)
+		c.Check(argText, Matches, `(?s).*--delete-keys\n8B48AD6246925553\n.*`)
+	})
+}
+
+// TestAPIGPGListKeysCommandFailure tests list error propagation from gpg
+func (s *GPGSuite) TestAPIGPGListKeysCommandFailure(c *C) {
+	script := "#!/bin/sh\n" +
+		"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
+		"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
+		"echo 'keyring missing'\n" +
+		"exit 1\n" +
+		"fi\n" +
+		"exit 1\n"
+
+	s.withFakeGPG(c, script, func(_ string) {
+		response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
+		c.Assert(err, IsNil)
+		c.Check(response.Code, Equals, 400)
+		c.Check(response.Body.String(), Matches, `(?s).*failed to list keys: keyring missing.*`)
+	})
+}
+
+// TestAPIGPGDeleteKeyCommandFailure tests delete error propagation from gpg
+func (s *GPGSuite) TestAPIGPGDeleteKeyCommandFailure(c *C) {
+	s.withFakeGPG(c, s.fakeGPGScript(c, "", "", "delete failed"), func(_ string) {
+		body, err := json.Marshal(gpgDeleteKeyParams{
+			Keyring: "trustedkeys.gpg",
+			GpgKeyID: "8B48AD6246925553",
+		})
+		c.Assert(err, IsNil)
+
+		response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
+		c.Assert(reqErr, IsNil)
+		c.Check(response.Code, Equals, 400)
+		c.Check(response.Body.String(), Matches, `(?s).*failed to delete key: delete failed.*`)
+	})
+}

api/metrics.go

@@ -102,7 +102,7 @@ func countPackagesByRepos() {
 
 		components := repo.Components()
 		for _, c := range components {
-			count := float64(repo.RefList(c).Len())
+			count := float64(len(repo.RefList(c).Refs))
 			apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
 		}
 

api/middleware_test.go

@@ -67,17 +67,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "warn")
@@ -130,17 +130,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "info")
@@ -153,17 +153,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/ready", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	if val, ok := jsonMap["level"]; ok {
 		c.Check(val, Equals, "error")
@@ -176,17 +176,17 @@ func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
 	outC := make(chan string)
 	go func() {
 		var buf bytes.Buffer
-		io.Copy(&buf, s.logReader)
+		_, _ = io.Copy(&buf, s.logReader)
 		fmt.Println(buf.String())
 		outC <- buf.String()
 	}()
 
 	s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
-	s.logWriter.Close()
+	_ = s.logWriter.Close()
 	capturedOutput := <-outC
 
 	var jsonMap map[string]interface{}
-	json.Unmarshal([]byte(capturedOutput), &jsonMap)
+	_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
 
 	fmt.Println(capturedOutput)
 

api/mirror.go

@@ -31,22 +31,61 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
 	return verifier, nil
 }
 
+// stringSlicesEqual compares two string slices for equality (order matters)
+func stringSlicesEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
+// uniqueStrings returns a new slice with only unique strings from the input, sorted
+func uniqueStrings(input []string) []string {
+	if len(input) == 0 {
+		return input
+	}
+	seen := make(map[string]struct{}, len(input))
+	result := make([]string, 0, len(input))
+	for _, s := range input {
+		if _, ok := seen[s]; !ok {
+			seen[s] = struct{}{}
+			result = append(result, s)
+		}
+	}
+	sort.Strings(result)
+	return result
+}
+
 // @Summary List Mirrors
 // @Description **Show list of currently available mirrors**
 // @Description Each mirror is returned as in “show” API.
 // @Tags Mirrors
 // @Produce json
-// @Success 200 {array} deb.RemoteRepo
+// @Success 200 {array} remoteRepoResponse
 // @Router /api/mirrors [get]
 func apiMirrorsList(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
-	result := []*deb.RemoteRepo{}
+	result := []remoteRepoResponse{}
-	collection.ForEach(func(repo *deb.RemoteRepo) error {
+	err := collection.ForEach(func(repo *deb.RemoteRepo) error {
-		result = append(result, repo)
+		err := collection.LoadComplete(repo)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newRemoteRepoResponse(repo))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
+	}
 
 	c.JSON(200, result)
 }
@@ -72,6 +111,8 @@ type mirrorCreateParams struct {
 	DownloadUdebs bool `                     json:"DownloadUdebs"`
 	// Set "true" to mirror installer files
 	DownloadInstaller bool `                 json:"DownloadInstaller"`
+	// Set "true" to mirror AppStream (DEP-11) metadata
+	DownloadAppStream bool `                 json:"DownloadAppStream"`
 	// Set "true" to include dependencies of matching packages when filtering
 	FilterWithDeps bool `                    json:"FilterWithDeps"`
 	// Set "true" to skip if the given components are in the Release file
@@ -123,7 +164,7 @@ func apiMirrorsCreate(c *gin.Context) {
 	}
 
 	repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
-		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
+		b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream)
 
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
@@ -150,7 +191,7 @@ func apiMirrorsCreate(c *gin.Context) {
 		return
 	}
 
-	err = collection.Add(repo, collectionFactory.RefListCollection())
+	err = collection.Add(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
 		return
@@ -229,9 +270,10 @@ func apiMirrorsShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
 	}
 
 	c.JSON(200, repo)
@@ -260,7 +302,7 @@ func apiMirrorsPackages(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
 	}
@@ -319,7 +361,7 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 
 	if c.Request.URL.Query().Get("format") == "details" {
-		list.ForEach(func(p *deb.Package) error {
+		_ = list.ForEach(func(p *deb.Package) error {
 			result = append(result, p)
 			return nil
 		})
@@ -330,29 +372,133 @@ func apiMirrorsPackages(c *gin.Context) {
 	}
 }
 
+type mirrorEditParams struct {
+	// Package query that is applied to mirror packages
+	Filter *string `         json:"Filter" example:"xserver-xorg"`
+	// Set "true" to include dependencies of matching packages when filtering
+	FilterWithDeps *bool `   json:"FilterWithDeps"`
+	// Set "true" to mirror installer files
+	DownloadInstaller *bool `json:"DownloadInstaller"`
+	// Set "true" to mirror source packages
+	DownloadSources *bool `  json:"DownloadSources"`
+	// Set "true" to mirror udeb files
+	DownloadUdebs *bool `    json:"DownloadUdebs"`
+	// URL of the archive to mirror
+	ArchiveURL *string `     json:"ArchiveURL"     example:"http://deb.debian.org/debian"`
+	// Comma separated list of architectures
+	Architectures *[]string `json:"Architectures"  example:"amd64"`
+	// Gpg keyring(s) for verifying Release file if a mirror update is required.
+	Keyrings []string `      json:"Keyrings"       example:"trustedkeys.gpg"`
+	// Set "true" to skip the verification of Release file signatures
+	IgnoreSignatures *bool ` json:"IgnoreSignatures"`
+}
+
+// @Summary Edit Mirror
+// @Description **Edit mirror config**
+// @Tags Mirrors
+// @Param name path string true "mirror name to edit"
+// @Consume json
+// @Param request body mirrorEditParams true "Parameters"
+// @Produce json
+// @Success 200 {object} deb.RemoteRepo "Mirror was edited successfully"
+// @Failure 400 {object} Error "Bad Request"
+// @Failure 404 {object} Error "Mirror not found"
+// @Failure 409 {object} Error "Aptly db locked"
+// @Failure 500 {object} Error "Internal Error"
+// @Router /api/mirrors/{name} [post]
+func apiMirrorsEdit(c *gin.Context) {
+	var (
+		err  error
+		b    mirrorEditParams
+		repo *deb.RemoteRepo
+	)
+
+	collectionFactory := context.NewCollectionFactory()
+	collection := collectionFactory.RemoteRepoCollection()
+
+	name := c.Params.ByName("name")
+	repo, err = collection.ByName(name)
+	if err != nil {
+		AbortWithJSONError(c, 404, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	err = repo.CheckLock()
+	if err != nil {
+		AbortWithJSONError(c, 409, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	if c.Bind(&b) != nil {
+		return
+	}
+
+	fetchMirror := false
+	ignoreSignatures := context.Config().GpgDisableVerify
+
+	if b.Filter != nil {
+		repo.Filter = *b.Filter
+	}
+	if b.FilterWithDeps != nil {
+		repo.FilterWithDeps = *b.FilterWithDeps
+	}
+	if b.DownloadInstaller != nil {
+		repo.DownloadInstaller = *b.DownloadInstaller
+	}
+	if b.DownloadSources != nil {
+		repo.DownloadSources = *b.DownloadSources
+	}
+	if b.DownloadUdebs != nil {
+		repo.DownloadUdebs = *b.DownloadUdebs
+	}
+	if b.ArchiveURL != nil && *b.ArchiveURL != repo.ArchiveRoot {
+		repo.SetArchiveRoot(*b.ArchiveURL)
+		fetchMirror = true
+	}
+	if b.Architectures != nil {
+		uniqueArchitectures := uniqueStrings(*b.Architectures)
+		if !stringSlicesEqual(uniqueArchitectures, uniqueStrings(repo.Architectures)) {
+			repo.Architectures = uniqueArchitectures
+			fetchMirror = true
+		}
+	}
+	if b.IgnoreSignatures != nil {
+		ignoreSignatures = *b.IgnoreSignatures
+	}
+
+	if repo.IsFlat() && repo.DownloadUdebs {
+		AbortWithJSONError(c, 400, fmt.Errorf("unable to edit: flat mirrors don't support udebs"))
+		return
+	}
+
+	if fetchMirror {
+		verifier, err := getVerifier(b.Keyrings)
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG verifier: %s", err))
+			return
+		}
+
+		err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
+		if err != nil {
+			AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
+			return
+		}
+	}
+
+	err = collection.Update(repo)
+	if err != nil {
+		AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
+		return
+	}
+
+	c.JSON(200, repo)
+}
+
 type mirrorUpdateParams struct {
 	// Change mirror name to `Name`
 	Name string `                 json:"Name"                   example:"mirror1"`
-	// Url of the archive to mirror
-	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
-	// Package query that is applied to mirror packages
-	Filter string `               json:"Filter"                 example:"xserver-xorg"`
-	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
-	Architectures []string `      json:"Architectures"          example:"amd64"`
-	// Components to mirror, if not specified aptly would fetch all components
-	Components []string `         json:"Components"             example:"main"`
 	// Gpg keyring(s) for verifing Release file
 	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
-	// Set "true" to include dependencies of matching packages when filtering
-	FilterWithDeps bool `         json:"FilterWithDeps"`
-	// Set "true" to mirror source packages
-	DownloadSources bool `        json:"DownloadSources"`
-	// Set "true" to mirror udeb files
-	DownloadUdebs bool `          json:"DownloadUdebs"`
-	// Set "true" to skip checking if the given components are in the Release file
-	SkipComponentCheck bool `     json:"SkipComponentCheck"`
-	// Set "true" to skip checking if the given architectures are in the Release file
-	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
 	// Set "true" to ignore checksum errors
 	IgnoreChecksums bool `        json:"IgnoreChecksums"`
 	// Set "true" to skip the verification of Release file signatures
@@ -361,6 +507,8 @@ type mirrorUpdateParams struct {
 	ForceUpdate bool `            json:"ForceUpdate"`
 	// Set "true" to skip downloading already downloaded packages
 	SkipExistingPackages bool `   json:"SkipExistingPackages"`
+	// Set "true" to download only the latest version per package/architecture
+	LatestOnly bool `             json:"LatestOnly"`
 }
 
 // @Summary Update Mirror
@@ -394,14 +542,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 	}
 
 	b.Name = remote.Name
-	b.DownloadUdebs = remote.DownloadUdebs
-	b.DownloadSources = remote.DownloadSources
-	b.SkipComponentCheck = remote.SkipComponentCheck
-	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
-	b.FilterWithDeps = remote.FilterWithDeps
-	b.Filter = remote.Filter
-	b.Architectures = remote.Architectures
-	b.Components = remote.Components
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
 	log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -418,27 +558,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
-	if b.DownloadUdebs != remote.DownloadUdebs {
-		if remote.IsFlat() && b.DownloadUdebs {
-			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
-			return
-		}
-	}
-
-	if b.ArchiveURL != "" {
-		remote.SetArchiveRoot(b.ArchiveURL)
-	}
-
-	remote.Name = b.Name
-	remote.DownloadUdebs = b.DownloadUdebs
-	remote.DownloadSources = b.DownloadSources
-	remote.SkipComponentCheck = b.SkipComponentCheck
-	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
-	remote.FilterWithDeps = b.FilterWithDeps
-	remote.Filter = b.Filter
-	remote.Architectures = b.Architectures
-	remote.Components = b.Components
-
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -461,11 +580,19 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		if remote.DownloadAppStream && !remote.IsFlat() {
+			err = remote.DownloadAppStreamFiles(out, downloader,
+				context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+			}
+		}
+
 		if remote.Filter != "" {
 			var filterQuery deb.PackageQuery
 
@@ -481,7 +608,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -491,12 +618,12 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				collection.Update(remote, collectionFactory.RefListCollection())
+				_ = collection.Update(remote)
 			}
 		}()
 
 		remote.MarkAsUpdating()
-		err = collection.Update(remote, collectionFactory.RefListCollection())
+		err = collection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -579,7 +706,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 							file, e = os.CreateTemp("", task.File.Filename)
 							if e == nil {
 								task.TempDownPath = file.Name()
-								file.Close()
+								_ = file.Close()
 							}
 						}
 						if e != nil {
@@ -653,8 +780,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		log.Info().Msgf("%s: Finalizing download...", b.Name)
-		remote.FinalizeDownload(collectionFactory, out)
+		_ = remote.FinalizeDownload(collectionFactory, out)
-		err = collectionFactory.RemoteRepoCollection().Update(remote, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

api/mirror_test.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"encoding/json"
 
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/gin-gonic/gin"
 	. "gopkg.in/check.v1"
 )
 
 type MirrorSuite struct {
-	ApiSuite
+	APISuite
 }
 
 var _ = Suite(&MirrorSuite{})
@@ -17,7 +18,10 @@ var _ = Suite(&MirrorSuite{})
 func (s *MirrorSuite) TestGetMirrors(c *C) {
 	response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
 	c.Check(response.Code, Equals, 200)
-	c.Check(response.Body.String(), Equals, "[]")
+
+	var mirrors []map[string]interface{}
+	err := json.Unmarshal(response.Body.Bytes(), &mirrors)
+	c.Assert(err, IsNil)
 }
 
 func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
@@ -26,6 +30,21 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
 	c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
 }
 
+func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
+	body, err := json.Marshal(gin.H{
+		"Name":              "test-flat-appstream",
+		"ArchiveURL":        "http://example.com/repo/",
+		"Distribution":      "./",
+		"DownloadAppStream":  true,
+	})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Check(response.Code, Equals, 400)
+	c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
+}
+
 func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.ExpectFailure("Need to mock downloads")
 	body, err := json.Marshal(gin.H{
@@ -38,3 +57,49 @@ func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.Check(response.Code, Equals, 400)
 	c.Check(response.Body.String(), Equals, "")
 }
+
+func (s *MirrorSuite) TestGetMirrorsIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().RemoteRepoCollection()
+
+	repo, err := deb.NewRemoteRepo("count-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
+	c.Assert(err, IsNil)
+
+	err = collection.Add(repo)
+	c.Assert(err, IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), makePackageRefList(c).Encode())
+
+	response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var mirrors []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &mirrors)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, mirror := range mirrors {
+		if mirror["Name"] == "count-mirror" {
+			found = true
+			value, ok := mirror["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *MirrorSuite) TestGetMirrorsReturns500OnCorruptRefList(c *C) {
+	collection := s.context.NewCollectionFactory().RemoteRepoCollection()
+
+	repo, err := deb.NewRemoteRepo("broken-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
+	c.Assert(err, IsNil)
+	c.Assert(collection.Add(repo), IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
+
+	response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*unable to show:.*")
+}

api/package_count_response.go

@@ -0,0 +1,30 @@
+package api
+
+import "github.com/aptly-dev/aptly/deb"
+
+type remoteRepoResponse struct {
+	*deb.RemoteRepo
+	NumPackages int `json:"NumPackages"`
+}
+
+type localRepoResponse struct {
+	*deb.LocalRepo
+	NumPackages int `json:"NumPackages"`
+}
+
+type snapshotResponse struct {
+	*deb.Snapshot
+	NumPackages int `json:"NumPackages"`
+}
+
+func newRemoteRepoResponse(repo *deb.RemoteRepo) remoteRepoResponse {
+	return remoteRepoResponse{RemoteRepo: repo, NumPackages: repo.NumPackages()}
+}
+
+func newLocalRepoResponse(repo *deb.LocalRepo) localRepoResponse {
+	return localRepoResponse{LocalRepo: repo, NumPackages: repo.NumPackages()}
+}
+
+func newSnapshotResponse(snapshot *deb.Snapshot) snapshotResponse {
+	return snapshotResponse{Snapshot: snapshot, NumPackages: snapshot.NumPackages()}
+}

api/package_count_test_helpers_test.go

@@ -0,0 +1,19 @@
+package api
+
+import (
+	"github.com/aptly-dev/aptly/deb"
+	. "gopkg.in/check.v1"
+)
+
+func makePackageRefList(c *C) *deb.PackageRefList {
+	list := deb.NewPackageList()
+	c.Assert(list.Add(&deb.Package{Name: "libcount", Version: "1.0", Architecture: "amd64"}), IsNil)
+	c.Assert(list.Add(&deb.Package{Name: "appcount", Version: "2.0", Architecture: "all"}), IsNil)
+	return deb.NewPackageRefListFromPackageList(list)
+}
+
+func putRawDBValue(c *C, s *APISuite, key []byte, value []byte) {
+	db, err := s.context.Database()
+	c.Assert(err, IsNil)
+	c.Assert(db.Put(key, value), IsNil)
+}

api/packages_test.go

@@ -5,7 +5,7 @@ import (
 )
 
 type PackagesSuite struct {
-	ApiSuite
+	APISuite
 }
 
 var _ = Suite(&PackagesSuite{})

api/publish.go

@@ -16,8 +16,8 @@ import (
 type signingParams struct {
 	// Don't sign published repository
 	Skip bool `                json:"Skip"           example:"false"`
-	// GPG key ID to use when signing the release, if not specified default key is used
+	// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used
-	GpgKey string `            json:"GpgKey"         example:"A0546A43624A8331"`
+	GpgKey string `            json:"GpgKey"         example:"KEY_ID_a, KEY_ID_b"`
 	// GPG keyring to use (instead of default)
 	Keyring string `           json:"Keyring"        example:"trustedkeys.gpg"`
 	// GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -41,7 +41,21 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(options.GpgKey)
+
+	var multiGpgKeys []string
+	// REST params have priority over config
+	if options.GpgKey != "" {
+		for _, p := range strings.Split(options.GpgKey, ",") {
+			if t := strings.TrimSpace(p); t != "" {
+				multiGpgKeys = append(multiGpgKeys, t)
+			}
+		}
+	} else if len(context.Config().GpgKeys) > 0 {
+		multiGpgKeys = context.Config().GpgKeys
+	}
+	for _, gpgKey := range multiGpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
 
@@ -168,8 +182,12 @@ type publishedRepoCreateParams struct {
 	SkipBz2 *bool `                               json:"SkipBz2"               example:"false"`
 	// Provide index files by hash
 	AcquireByHash *bool `                         json:"AcquireByHash"         example:"false"`
+	// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file.
+	SignedBy *string `                            json:"SignedBy"              example:""`
 	// Enable multiple packages with the same filename in different distributions
 	MultiDist *bool `                             json:"MultiDist"             example:"false"`
+	// Version of the release
+	Version string `                              json:"Version"               example:""`
 }
 
 // @Summary Create Published Repository
@@ -297,10 +315,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			switch s := source.(type) {
 			case *deb.Snapshot:
 				snapshotCollection := collectionFactory.SnapshotCollection()
-				err = snapshotCollection.LoadComplete(s, collectionFactory.RefListCollection())
+				err = snapshotCollection.LoadComplete(s)
 			case *deb.LocalRepo:
 				localCollection := collectionFactory.LocalRepoCollection()
-				err = localCollection.LoadComplete(s, collectionFactory.RefListCollection())
+				err = localCollection.LoadComplete(s)
 			default:
 				err = fmt.Errorf("unexpected type for source: %T", source)
 			}
@@ -341,9 +359,17 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			published.AcquireByHash = *b.AcquireByHash
 		}
 
+		if b.SignedBy != nil {
+			published.SignedBy = *b.SignedBy
+		}
+
+		if b.Version != "" {
+			published.Version = b.Version
+		}
+
 		duplicate := collection.CheckDuplicate(published)
 		if duplicate != nil {
-			collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
@@ -352,7 +378,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		err = collection.Add(published, collectionFactory.RefListCollection())
+		err = collection.Add(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -376,8 +402,16 @@ type publishedRepoUpdateSwitchParams struct {
 	Snapshots []sourceParams `                    json:"Snapshots"`
 	// Provide index files by hash
 	AcquireByHash *bool `                         json:"AcquireByHash"  example:"false"`
+	// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
+	SignedBy *string `                            json:"SignedBy"  example:""`
 	// Enable multiple packages with the same filename in different distributions
 	MultiDist *bool `                             json:"MultiDist"      example:"false"`
+    // Value of Label: field in published repository stanza
+    Label *string `                               json:"Label"          example:"Debian"`
+    // Value of Origin: field in published repository stanza
+    Origin *string `                              json:"Origin"         example:"Debian"`
+    // Version of the release: Optional
+    Version *string `                             json:"Version"        example:"13.3"`
 }
 
 // @Summary Update Published Repository
@@ -393,7 +427,6 @@ type publishedRepoUpdateSwitchParams struct {
 // @Description
 // @Description See also: `aptly publish update` / `aptly publish switch`
 // @Tags Publish
-// @Produce json
 // @Param prefix path string true "publishing prefix"
 // @Param distribution path string true "distribution name"
 // @Param _async query bool false "Run in background and return task object"
@@ -462,16 +495,32 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		published.AcquireByHash = *b.AcquireByHash
 	}
 
+	if b.SignedBy != nil {
+		published.SignedBy = *b.SignedBy
+	}
+
 	if b.MultiDist != nil {
 		published.MultiDist = *b.MultiDist
 	}
 
+	if b.Label != nil {
+		published.Label = *b.Label
+	}
+
+	if b.Origin != nil {
+		published.Origin = *b.Origin
+	}
+
+	if b.Version != nil {
+		published.Version = *b.Version
+	}
+
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(published, collectionFactory, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(published, collectionFactory)
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
 		revision := published.ObtainRevision()
@@ -487,15 +536,15 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 
 		result, err := published.Update(collectionFactory, out)
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
 		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published, collectionFactory.RefListCollection())
+		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -625,7 +674,7 @@ func apiPublishAddSource(c *gin.Context) {
 
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -739,7 +788,7 @@ func apiPublishSetSources(c *gin.Context) {
 
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -789,7 +838,7 @@ func apiPublishDropChanges(c *gin.Context) {
 
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -869,7 +918,7 @@ func apiPublishUpdateSource(c *gin.Context) {
 
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -932,7 +981,7 @@ func apiPublishRemoveSource(c *gin.Context) {
 
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
-	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		err = collection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -955,8 +1004,16 @@ type publishedRepoUpdateParams struct {
 	SkipCleanup *bool `                           json:"SkipCleanup"     example:"false"`
 	// Provide index files by hash
 	AcquireByHash *bool `                         json:"AcquireByHash"   example:"false"`
+	// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
+	SignedBy *string `                            json:"SignedBy"   example:""`
 	// Enable multiple packages with the same filename in different distributions
 	MultiDist *bool `                             json:"MultiDist"       example:"false"`
+    // Value of Label: field in published repository stanza
+    Label *string `                               json:"Label"          example:"Debian"`
+    // Value of Origin: field in published repository stanza
+    Origin *string `                              json:"Origin"         example:"Debian"`
+    // Version of the release: Optional
+    Version *string `                             json:"Version"        example:"13.3"`
 }
 
 // @Summary Update Published Repository
@@ -1021,10 +1078,26 @@ func apiPublishUpdate(c *gin.Context) {
 		published.AcquireByHash = *b.AcquireByHash
 	}
 
+	if b.SignedBy != nil {
+		published.SignedBy = *b.SignedBy
+	}
+
 	if b.MultiDist != nil {
 		published.MultiDist = *b.MultiDist
 	}
 
+	if b.Label != nil {
+		published.Label = *b.Label
+	}
+
+	if b.Origin != nil {
+		published.Origin = *b.Origin
+	}
+
+	if b.Version != nil {
+		published.Version = *b.Version
+	}
+
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {

api/repos.go

@@ -24,19 +24,19 @@ import (
 // @Tags Repos
 // @Produce html
 // @Success 200 {object} string "HTML"
-// @Router /api/repos [get]
+// @Router /repos [get]
 func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
 		c.Writer.Flush()
-		c.Writer.WriteString("<pre>\n")
+		_, _ = c.Writer.WriteString("<pre>\n")
 		if len(localRepos) == 0 {
-			c.Writer.WriteString("<a href=\"-/\">default</a>\n")
+			_, _ = c.Writer.WriteString("<a href=\"-/\">default</a>\n")
 		}
 		for publishPrefix := range localRepos {
-			c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
+			_, _ = c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
 		}
-		c.Writer.WriteString("</pre>")
+		_, _ = c.Writer.WriteString("</pre>")
 		c.Writer.Flush()
 	}
 }
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
 // @Param pkgPath path string true "Package Path" allowReserved=true
 // @Produce json
 // @Success 200 ""
-// @Router /api/{storage}/{pkgPath} [get]
+// @Router /repos/{storage}/{pkgPath} [get]
 func reposServeInAPIMode(c *gin.Context) {
 	pkgpath := c.Param("pkgPath")
 
@@ -69,17 +69,26 @@ func reposServeInAPIMode(c *gin.Context) {
 // @Description Each repo is returned as in “show” API.
 // @Tags Repos
 // @Produce  json
-// @Success 200 {array} deb.LocalRepo
+// @Success 200 {array} localRepoResponse
 // @Router /api/repos [get]
 func apiReposList(c *gin.Context) {
-	result := []*deb.LocalRepo{}
+	result := []localRepoResponse{}
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	collection.ForEach(func(r *deb.LocalRepo) error {
+	err := collection.ForEach(func(r *deb.LocalRepo) error {
-		result = append(result, r)
+		err := collection.LoadComplete(r)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newLocalRepoResponse(r))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
 
 	c.JSON(200, result)
 }
@@ -107,9 +116,9 @@ type repoCreateParams struct {
 // @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
 // @Description ```
 // @Tags Repos
-// @Produce  json
 // @Consume  json
 // @Param request body repoCreateParams true "Parameters"
+// @Produce  json
 // @Success 201 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Source snapshot not found"
 // @Failure 409 {object} Error "Local repo already exists"
@@ -155,13 +164,6 @@ func apiReposCreate(c *gin.Context) {
 		return
 	}
 
-	collection := collectionFactory.LocalRepoCollection()
-	err := collection.Add(repo, collectionFactory.RefListCollection())
-	if err != nil {
-		AbortWithJSONError(c, 400, err)
-		return
-	}
-
 	err := localRepoCollection.Add(repo)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusInternalServerError, err)
@@ -185,8 +187,10 @@ type reposEditParams struct {
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposEditParams true "Parameters"
+// @Produce json
 // @Success 200 {object} deb.LocalRepo "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 500 {object} Error "Internal Server Error"
@@ -200,17 +204,18 @@ func apiReposEdit(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
-	if b.Name != nil {
+	if b.Name != nil && *b.Name != name {
 		_, err := collection.ByName(*b.Name)
 		if err == nil {
 			// already exists
-			AbortWithJSONError(c, 404, err)
+			AbortWithJSONError(c, 404, fmt.Errorf("local repo with name %q already exists", *b.Name))
 			return
 		}
 		repo.Name = *b.Name
@@ -225,7 +230,7 @@ func apiReposEdit(c *gin.Context) {
 		repo.DefaultComponent = *b.DefaultComponent
 	}
 
-	err = collection.Update(repo, collectionFactory.RefListCollection())
+	err = collection.Update(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -238,8 +243,8 @@ func apiReposEdit(c *gin.Context) {
 // @Summary Get Repository Info
 // @Description Returns basic information about local repository.
 // @Tags Repos
-// @Produce  json
 // @Param name path string true "Repository name"
+// @Produce  json
 // @Success 200 {object} deb.LocalRepo
 // @Failure 404 {object} Error "Repository not found"
 // @Router /api/repos/{name} [get]
@@ -261,9 +266,10 @@ func apiReposShow(c *gin.Context) {
 // @Description Cannot drop repos that are published.
 // @Description Needs force=1 to drop repos used as source by other repos.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Param force query int false "force: 1 to enable"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "Repo object"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Repo Conflict"
@@ -313,12 +319,12 @@ func apiReposDrop(c *gin.Context) {
 // @Description ["Pi386 aptly 0.8 966561016b44ed80"]
 // @Description ```
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
-// @Param name path string true "Snapshot to search"
 // @Param q query string true "Package query (e.g Name%20(~%20matlab))"
 // @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
 // @Param format query string true "Set to 'details' to return extra info about each package"
 // @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Failure 404 {object} Error "Internal Server Error"
@@ -333,7 +339,7 @@ func apiReposPackagesShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -367,7 +373,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	resources := []string{string(repo.Key())}
 
 	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -396,9 +402,9 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 			}
 		}
 
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -413,9 +419,11 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -433,9 +441,11 @@ func apiReposPackagesAdd(c *gin.Context) {
 // @Description
 // @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
-// @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
+// @Consume  json
+// @Param request body reposPackagesAddDeleteParams true "Parameters"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -456,7 +466,7 @@ func apiReposPackagesDelete(c *gin.Context) {
 // @Tags Repos
 // @Param name path string true "Repository name"
 // @Param dir path string true "Directory of packages"
-// @Param file path string false "Filename (optional)"
+// @Param file path string true "Filename"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 200 {string} string "OK"
@@ -524,7 +534,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -559,9 +569,9 @@ func apiReposPackageFromDir(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import package files: %s", err)
 		}
 
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -577,7 +587,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 			}
 
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -614,11 +624,11 @@ type reposCopyPackageParams struct {
 // @Summary Copy Package
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Destination repo"
-// @Param name path string true "Source repo"
+// @Param src path string true "Source repo"
-// @Param src path string true "Destination repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -769,12 +779,15 @@ func apiReposCopyPackage(c *gin.Context) {
 // @Summary Include File from Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
+// @Param file path string true "File/packages to include"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} string "msg"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir}/{file} [post]
@@ -783,26 +796,22 @@ func apiReposIncludePackageFromFile(c *gin.Context) {
 	apiReposIncludePackageFromDir(c)
 }
 
-type reposIncludePackageFromDirReport struct {
-	Warnings []string
-	Added    []string
-	Deleted  []string
-}
-
 type reposIncludePackageFromDirResponse struct {
-	Report      reposIncludePackageFromDirReport
+	Report      *aptly.RecordingResultReporter
 	FailedFiles []string
 }
 
 // @Summary Include Directory
 // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
 // @Tags Repos
-// @Produce json
+// @Param name path string true "Repository name"
+// @Param dir path string true "Directory of packages"
 // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
 // @Param noRemoveFiles query int false "when value is set to 1, don’t remove files that have been imported successfully into repository"
 // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
 // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} reposIncludePackageFromDirResponse "Response"
 // @Failure 404 {object} Error "Not Found"
 // @Router /api/repos/{name}/include/{dir} [post]
@@ -843,7 +852,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	}
 
 	var resources []string
-	if len(repoTemplate.Tree.Root.Nodes) > 1 {
+	if len(repoTemplate.Root.Nodes) > 1 {
 		resources = append(resources, task.AllLocalReposResourcesKey)
 	} else {
 		// repo template string is simple text so only use resource key of specific repository
@@ -874,7 +883,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		_, failedFiles2, err = deb.ImportChangesFiles(
 			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
 			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-			collectionFactory.RefListCollection(), context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
 		failedFiles = append(failedFiles, failedFiles2...)
 
 		if err != nil {
@@ -883,7 +892,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 
 		if !noRemoveFiles {
 			// atempt to remove dir, if it fails, that's fine: probably it's not empty
-			os.Remove(filepath.Join(context.UploadPath(), dirParam))
+			_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
 		}
 
 		if failedFiles == nil {
@@ -903,9 +912,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 			out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
+		ret := reposIncludePackageFromDirResponse{
-			"Report":      reporter,
+			Report:      reporter,
-			"FailedFiles": failedFiles,
+			FailedFiles: failedFiles,
-		}}, nil
+		}
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
 	})
 }

api/repos_test.go

@@ -0,0 +1,63 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	. "gopkg.in/check.v1"
+)
+
+type ReposSuite struct {
+	APISuite
+}
+
+var _ = Suite(&ReposSuite{})
+
+func (s *ReposSuite) TestGetReposIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().LocalRepoCollection()
+	repo := deb.NewLocalRepo("count-repo-list", "")
+	repo.UpdateRefList(makePackageRefList(c))
+	c.Assert(collection.Add(repo), IsNil)
+
+	response, err := s.HTTPRequest("GET", "/api/repos", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var repos []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &repos)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, repo := range repos {
+		if repo["Name"] == "count-repo-list" {
+			found = true
+			value, ok := repo["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *ReposSuite) TestGetReposReturns500OnCorruptRefList(c *C) {
+	body, err := json.Marshal(gin.H{"Name": "broken-repo-list"})
+	c.Assert(err, IsNil)
+
+	response, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 201)
+
+	collection := s.context.NewCollectionFactory().LocalRepoCollection()
+	repo, err := collection.ByName("broken-repo-list")
+	c.Assert(err, IsNil)
+	putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
+
+	response, err = s.HTTPRequest("GET", "/api/repos", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
+}

api/router.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"net/http"
-	"os"
 	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -19,6 +18,12 @@ import (
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		countPackagesByRepos()
@@ -56,13 +61,9 @@ func Router(c *ctx.AptlyContext) http.Handler {
 	router.UseRawPath = true
 
 	if c.Config().LogFormat == "json" {
-		c.StructuredLogging(true)
-		utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
 		gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
 		router.Use(JSONLogger())
 	} else {
-		c.StructuredLogging(false)
-		utils.SetupDefaultLogger(c.Config().LogLevel)
 		router.Use(gin.Logger())
 	}
 
@@ -163,12 +164,15 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.GET("/mirrors/:name", apiMirrorsShow)
 		api.GET("/mirrors/:name/packages", apiMirrorsPackages)
 		api.POST("/mirrors", apiMirrorsCreate)
+		api.POST("/mirrors/:name", apiMirrorsEdit)
 		api.PUT("/mirrors/:name", apiMirrorsUpdate)
 		api.DELETE("/mirrors/:name", apiMirrorsDrop)
 	}
 
 	{
+		api.GET("/gpg/keys", apiGPGListKeys)
 		api.POST("/gpg/key", apiGPGAddKey)
+		api.DELETE("/gpg/key", apiGPGDeleteKey)
 	}
 
 	{
@@ -220,7 +224,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		api.GET("/graph.:ext", apiGraph)
 	}
 	{
-		api.POST("/db/cleanup", apiDbCleanup)
+		api.POST("/db/cleanup", apiDBCleanup)
 	}
 	{
 		api.GET("/tasks", apiTasksList)

api/snapshot.go

@@ -20,7 +20,7 @@ import (
 // @Description Each snapshot is returned as in “show” API.
 // @Tags Snapshots
 // @Produce  json
-// @Success 200 {array} deb.Snapshot
+// @Success 200 {array} snapshotResponse
 // @Router /api/snapshots [get]
 func apiSnapshotsList(c *gin.Context) {
 	SortMethodString := c.Request.URL.Query().Get("sort")
@@ -32,11 +32,20 @@ func apiSnapshotsList(c *gin.Context) {
 		SortMethodString = "name"
 	}
 
-	result := []*deb.Snapshot{}
+	result := []snapshotResponse{}
-	collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
+	err := collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
-		result = append(result, snapshot)
+		err := collection.LoadComplete(snapshot)
+		if err != nil {
+			return err
+		}
+
+		result = append(result, newSnapshotResponse(snapshot))
 		return nil
 	})
+	if err != nil {
+		AbortWithJSONError(c, 500, err)
+		return
+	}
 
 	c.JSON(200, result)
 }
@@ -93,7 +102,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
 		}
 
-		err = collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -107,7 +116,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -174,7 +183,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 
 	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
 		for i := range sources {
-			err = snapshotCollection.LoadComplete(sources[i], collectionFactory.RefListCollection())
+			err = snapshotCollection.LoadComplete(sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
@@ -197,9 +206,9 @@ func apiSnapshotsCreate(c *gin.Context) {
 			}
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewSplitRefListFromPackageList(list), b.Description)
+		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -217,10 +226,9 @@ type snapshotsCreateFromRepositoryParams struct {
 // @Summary Snapshot Repository
 // @Description **Create a snapshot of a repository by name**
 // @Tags Snapshots
-// @Param name path string true "Repository name"
 // @Consume json
 // @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
-// @Param name path string true "Name of the snapshot"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -255,7 +263,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	resources := []string{string(repo.Key()), "S" + b.Name}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.LoadComplete(repo, collectionFactory.RefListCollection())
+		err := collection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -269,7 +277,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -333,7 +341,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = collectionFactory.SnapshotCollection().Update(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Update(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -360,7 +368,7 @@ func apiSnapshotsShow(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshot)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -452,20 +460,20 @@ func apiSnapshotsDiff(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshotA, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshotA)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
 	}
 
-	err = collection.LoadComplete(snapshotB, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshotB)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection(), nil)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -508,7 +516,7 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
 		return
 	}
 
-	err = collection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collection.LoadComplete(snapshot)
 	if err != nil {
 		AbortWithJSONError(c, 500, err)
 		return
@@ -555,7 +563,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 	}
 
 	if len(body.Sources) < 1 {
-		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
+		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("minimum one source snapshot is required"))
 		return
 	}
 
@@ -584,13 +592,13 @@ func apiSnapshotsMerge(c *gin.Context) {
 	}
 
 	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = snapshotCollection.LoadComplete(sources[0], collectionFactory.RefListCollection())
+		err = snapshotCollection.LoadComplete(sources[0])
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 		result := sources[0].RefList()
 		for i := 1; i < len(sources); i++ {
-			err = snapshotCollection.LoadComplete(sources[i], collectionFactory.RefListCollection())
+			err = snapshotCollection.LoadComplete(sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
@@ -609,7 +617,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -765,7 +773,7 @@ func apiSnapshotsPull(c *gin.Context) {
 		addedPackages := []string{}
 		alreadySeen := map[string]bool{}
 
-		destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
+		_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
 			key := pkg.Architecture + "_" + pkg.Name
 			_, seen := alreadySeen[key]
 
@@ -781,7 +789,7 @@ func apiSnapshotsPull(c *gin.Context) {
 
 			// If !allMatches, add only first matching name-arch package
 			if !seen || allMatches {
-				toPackageList.Add(pkg)
+				_ = toPackageList.Add(pkg)
 				addedPackages = append(addedPackages, pkg.String())
 			}
 

api/snapshot_test.go

@@ -0,0 +1,53 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/aptly-dev/aptly/deb"
+	. "gopkg.in/check.v1"
+)
+
+type SnapshotsSuite struct {
+	APISuite
+}
+
+var _ = Suite(&SnapshotsSuite{})
+
+func (s *SnapshotsSuite) TestGetSnapshotsIncludesNumPackages(c *C) {
+	collection := s.context.NewCollectionFactory().SnapshotCollection()
+	snapshot := deb.NewSnapshotFromRefList("count-snapshot-list", nil, makePackageRefList(c), "")
+	c.Assert(collection.Add(snapshot), IsNil)
+
+	response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 200)
+
+	var snapshots []map[string]interface{}
+	err = json.Unmarshal(response.Body.Bytes(), &snapshots)
+	c.Assert(err, IsNil)
+
+	found := false
+	for _, snapshot := range snapshots {
+		if snapshot["Name"] == "count-snapshot-list" {
+			found = true
+			value, ok := snapshot["NumPackages"]
+			c.Assert(ok, Equals, true)
+			c.Assert(value, Equals, float64(2))
+			break
+		}
+	}
+
+	c.Assert(found, Equals, true)
+}
+
+func (s *SnapshotsSuite) TestGetSnapshotsReturns500OnCorruptRefList(c *C) {
+	collection := s.context.NewCollectionFactory().SnapshotCollection()
+	snapshot := deb.NewSnapshotFromRefList("broken-snapshot-list", nil, makePackageRefList(c), "")
+	c.Assert(collection.Add(snapshot), IsNil)
+	putRawDBValue(c, &s.APISuite, snapshot.RefKey(), []byte("not-msgpack"))
+
+	response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
+	c.Assert(err, IsNil)
+	c.Assert(response.Code, Equals, 500)
+	c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
+}

aptly/conf.go

@@ -1,4 +1,4 @@
 package aptly
 
-// Default aptly.conf (filled in at link time)
+// AptlyConf holds the default aptly.conf (filled in at link time)
 var AptlyConf []byte

azure/package_pool.go

@@ -29,7 +29,7 @@ func NewPackagePool(accountName, accountKey, container, prefix, endpoint string)
 	return &PackagePool{az: azctx}, nil
 }
 
-// String
+// String returns the storage as string
 func (pool *PackagePool) String() string {
 	return pool.az.String()
 }
@@ -104,7 +104,7 @@ func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
 	}
-	defer os.Remove(temp.Name())
+	defer func() { _ = os.Remove(temp.Name()) }()
 
 	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
 	if err != nil {
@@ -156,7 +156,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	if err != nil {
 		return "", err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
 	err = pool.az.putFile(path, source, checksums.MD5)
 	if err != nil {

azure/package_pool_test.go

@@ -2,12 +2,12 @@ package azure
 
 import (
 	"context"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
 	"runtime"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
@@ -50,10 +50,10 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
 
 	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
+	publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
+	_, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
+		Access: &publicAccessType,
-        })
+	})
 	c.Assert(err, IsNil)
 
 	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -69,8 +69,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
 	c.Check(err, IsNil)
 	c.Check(list, DeepEquals, []string{})
 
-	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
-	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
 
 	list, err = s.pool.FilepathList(nil)
 	c.Check(err, IsNil)
@@ -81,8 +81,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
 }
 
 func (s *PackagePoolSuite) TestRemove(c *C) {
-	s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
-	s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
+	_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
 
 	size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
 	c.Check(err, IsNil)
@@ -247,7 +247,7 @@ func (s *PackagePoolSuite) TestOpen(c *C) {
 
 	f, err := s.pool.Open(path)
 	c.Assert(err, IsNil)
-	contents, err := ioutil.ReadAll(f)
+	contents, err := io.ReadAll(f)
 	c.Assert(err, IsNil)
 	c.Check(len(contents), Equals, 2738)
 	c.Check(f.Close(), IsNil)

azure/public.go

@@ -18,7 +18,7 @@ import (
 
 // PublishedStorage abstract file system with published files (actually hosted on Azure)
 type PublishedStorage struct {
-	prefix    string
+	// FIXME: unused ???? prefix    string
 	az        *azContext
 	pathCache map[string]map[string]string
 }
@@ -38,7 +38,7 @@ func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint st
 	return &PublishedStorage{az: azctx}, nil
 }
 
-// String
+// String returns the storage as string
 func (storage *PublishedStorage) String() string {
 	return storage.az.String()
 }
@@ -65,7 +65,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	if err != nil {
 		return err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
 	err = storage.az.putFile(path, source, sourceMD5)
 	if err != nil {
@@ -158,7 +158,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	if err != nil {
 		return err
 	}
-	defer source.Close()
+	defer func() { _ = source.Close() }()
 
 	err = storage.az.putFile(relFilePath, source, sourceMD5)
 	if err == nil {
@@ -193,7 +193,9 @@ func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadat
 	if err != nil {
 		return fmt.Errorf("error acquiring lease on source blob %s", src)
 	}
-	defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
+	defer func() {
+		_, _ = blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
+	}()
 
 	dstBlobClient := containerClient.NewBlobClient(dst)
 	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{

azure/public_test.go

@@ -1,17 +1,17 @@
 package azure
 
 import (
+	"bytes"
 	"context"
 	"crypto/md5"
 	"crypto/rand"
-	"io/ioutil"
+	"io"
 	"os"
 	"path/filepath"
-        "bytes"
 
-        "github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-        "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 	. "gopkg.in/check.v1"
@@ -36,7 +36,7 @@ func randString(n int) string {
 	}
 	const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
 	var bytes = make([]byte, n)
-	rand.Read(bytes)
+	_, _ = rand.Read(bytes)
 	for i, b := range bytes {
 		bytes[i] = alphanum[b%byte(len(alphanum))]
 	}
@@ -69,10 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-        publicAccessType := azblob.PublicAccessTypeContainer
+	publicAccessType := azblob.PublicAccessTypeContainer
-        _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
+	_, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
-            Access: &publicAccessType,
+		Access: &publicAccessType,
-        })
+	})
 	c.Assert(err, IsNil)
 
 	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -80,39 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-        _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	_, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-        resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
 	c.Assert(err, IsNil)
-	data, err := ioutil.ReadAll(resp.Body)
+	data, err := io.ReadAll(resp.Body)
 	c.Assert(err, IsNil)
 	return data
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-        serviceClient := s.storage.az.client.ServiceClient()
+	serviceClient := s.storage.az.client.ServiceClient()
-        containerClient := serviceClient.NewContainerClient(s.storage.az.container)
+	containerClient := serviceClient.NewContainerClient(s.storage.az.container)
-        blobClient := containerClient.NewBlobClient(path)
+	blobClient := containerClient.NewBlobClient(path)
-        _, err := blobClient.GetProperties(context.Background(), nil)
+	_, err := blobClient.GetProperties(context.Background(), nil)
 	c.Assert(err, NotNil)
 
-        storageError, ok := err.(*azcore.ResponseError)
+	storageError, ok := err.(*azcore.ResponseError)
 	c.Assert(ok, Equals, true)
 	c.Assert(storageError.StatusCode, Equals, 404)
 }
 
 func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	hash := md5.Sum(data)
-        uploadOptions := &azblob.UploadStreamOptions{
+	uploadOptions := &azblob.UploadStreamOptions{
-            HTTPHeaders: &blob.HTTPHeaders{
+		HTTPHeaders: &blob.HTTPHeaders{
-            	BlobContentMD5: hash[:],
+			BlobContentMD5: hash[:],
-            },
+		},
-        }
+	}
-        reader := bytes.NewReader(data)
+	reader := bytes.NewReader(data)
-        _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	_, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
 	c.Assert(err, IsNil)
 }
 
@@ -121,7 +121,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
 	filename := "a/b.txt"
 
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -140,7 +140,7 @@ func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
 	filename := "a/b+c.txt"
 
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -258,7 +258,7 @@ func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
 
 func (s *PublishedStorageSuite) TestRenameFile(c *C) {
 	dir := c.MkDir()
-	err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
+	err := os.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
 	c.Assert(err, IsNil)
 
 	err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
@@ -280,18 +280,18 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 	cs := files.NewMockChecksumStorage()
 
 	tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
-	err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
+	err := os.WriteFile(tmpFile1, []byte("Contents"), 0644)
 	c.Assert(err, IsNil)
 	cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
 
 	tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
-	err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
+	err = os.WriteFile(tmpFile2, []byte("Spam"), 0644)
 	c.Assert(err, IsNil)
 	cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
 
 	tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
-	os.MkdirAll(filepath.Dir(tmpFile3), 0777)
+	_ = os.MkdirAll(filepath.Dir(tmpFile3), 0777)
-	err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
+	err = os.WriteFile(tmpFile3, []byte("Contents"), 0644)
 	c.Assert(err, IsNil)
 	cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
 

cmd/api_serve.go

@@ -46,7 +46,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	}
 	if err == nil && len(listeners) == 1 {
 		listener := listeners[0]
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 		fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
 		err = http.Serve(listener, api.Router(context))
 		if err != nil {
@@ -67,7 +67,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 		if _, ok := <-sigchan; ok {
 			fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
 			context.TaskList().Wait()
-			server.Shutdown(stdcontext.Background())
+			_ = server.Shutdown(stdcontext.Background())
 		}
 	})()
 	defer close(sigchan)
@@ -75,14 +75,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
 	listenURL, err := url.Parse(listen)
 	if err == nil && listenURL.Scheme == "unix" {
 		file := listenURL.Path
-		os.Remove(file)
+		_ = os.Remove(file)
 
 		var listener net.Listener
 		listener, err = net.Listen("unix", file)
 		if err != nil {
 			return fmt.Errorf("failed to listen on: %s\n%s", file, err)
 		}
-		defer listener.Close()
+		defer func() { _ = listener.Close() }()
 
 		err = server.Serve(listener)
 	} else {

cmd/cmd.go

@@ -21,7 +21,7 @@ const (
 )
 
 // ListPackagesRefList shows list of packages in PackageRefList
-func ListPackagesRefList(reflist deb.AnyRefList, collectionFactory *deb.CollectionFactory) (err error) {
+func ListPackagesRefList(reflist *deb.PackageRefList, collectionFactory *deb.CollectionFactory) (err error) {
 	fmt.Printf("Packages:\n")
 
 	if reflist == nil {
@@ -97,7 +97,7 @@ package environment to new version.`,
 		Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
 		Subcommands: []*commander.Command{
 			makeCmdConfig(),
-			makeCmdDb(),
+			makeCmdDB(),
 			makeCmdGraph(),
 			makeCmdMirror(),
 			makeCmdRepo(),

cmd/config_show.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 
 	"github.com/smira/commander"
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func aptlyConfigShow(_ *commander.Command, _ []string) error {

cmd/db.go

@@ -4,13 +4,13 @@ import (
 	"github.com/smira/commander"
 )
 
-func makeCmdDb() *commander.Command {
+func makeCmdDB() *commander.Command {
 	return &commander.Command{
 		UsageLine: "db",
 		Short:     "manage aptly's internal database and package pool",
 		Subcommands: []*commander.Command{
-			makeCmdDbCleanup(),
+			makeCmdDBCleanup(),
-			makeCmdDbRecover(),
+			makeCmdDBRecover(),
 		},
 	}
 }

cmd/db_cleanup.go

@@ -6,14 +6,13 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
 	"github.com/aptly-dev/aptly/deb"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/smira/commander"
 )
 
 // aptly db cleanup
-func aptlyDbCleanup(cmd *commander.Command, args []string) error {
+func aptlyDBCleanup(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -25,20 +24,13 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	dryRun := context.Flags().Lookup("dry-run").Value.Get().(bool)
 	collectionFactory := context.NewCollectionFactory()
 
-	// collect information about references packages and their reflistbuckets...
+	// collect information about references packages...
-	existingPackageRefs := deb.NewSplitRefList()
+	existingPackageRefs := deb.NewPackageRefList()
-	existingBuckets := deb.NewRefListDigestSet()
+	referencedAppStreamFiles := []string{}
 
 	// used only in verbose mode to report package use source
 	packageRefSources := map[string][]string{}
 
-	var reflistMigration *deb.RefListMigration
-	if !dryRun {
-		reflistMigration = collectionFactory.RefListCollection().NewMigration()
-	} else {
-		reflistMigration = collectionFactory.RefListCollection().NewMigrationDryRun()
-	}
-
 	context.Progress().ColoredPrintf("@{w!}Loading mirrors, local repos, snapshots and published repos...@|")
 	if verbose {
 		context.Progress().ColoredPrintf("@{y}Loading mirrors:@|")
@@ -48,21 +40,24 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		sl := deb.NewSplitRefList()
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
+		if e != nil {
-		if e != nil && e != database.ErrNotFound {
 			return e
 		}
+		if repo.RefList() != nil {
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+			if verbose {
-		existingBuckets.AddAllInRefList(sl)
+				description := fmt.Sprintf("mirror %s", repo.Name)
+				_ = repo.RefList().ForEach(func(key []byte) error {
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+					return nil
+				})
+			}
+		}
 
-		if verbose {
+		for _, poolPath := range repo.AppStreamFiles {
-			description := fmt.Sprintf("mirror %s", repo.Name)
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
-			sl.ForEach(func(key []byte) error {
-				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
-				return nil
-			})
 		}
 
 		return nil
@@ -81,23 +76,21 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", repo.Name)
 		}
 
-		sl := deb.NewSplitRefList()
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, repo.RefKey(), reflistMigration)
+		if e != nil {
-		if e != nil && e != database.ErrNotFound {
 			return e
 		}
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+		if repo.RefList() != nil {
-		existingBuckets.AddAllInRefList(sl)
+			existingPackageRefs = existingPackageRefs.Merge(repo.RefList(), false, true)
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+			if verbose {
-
+				description := fmt.Sprintf("local repo %s", repo.Name)
-		if verbose {
+				_ = repo.RefList().ForEach(func(key []byte) error {
-			description := fmt.Sprintf("local repo %s", repo.Name)
+					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
-			sl.ForEach(func(key []byte) error {
+					return nil
-				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
+				})
-				return nil
+			}
-			})
 		}
 
 		return nil
@@ -116,22 +109,25 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("- @{g}%s@|", snapshot.Name)
 		}
 
-		sl := deb.NewSplitRefList()
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
-		e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, snapshot.RefKey(), reflistMigration)
 		if e != nil {
 			return e
 		}
 
-		existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
+		existingPackageRefs = existingPackageRefs.Merge(snapshot.RefList(), false, true)
-		existingBuckets.AddAllInRefList(sl)
 
 		if verbose {
 			description := fmt.Sprintf("snapshot %s", snapshot.Name)
-			sl.ForEach(func(key []byte) error {
+			_ = snapshot.RefList().ForEach(func(key []byte) error {
 				packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 				return nil
 			})
 		}
+
+		for _, poolPath := range snapshot.AppStreamFiles {
+			referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
+		}
+
 		return nil
 	})
 	if err != nil {
@@ -150,21 +146,17 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		if published.SourceKind != deb.SourceLocalRepo {
 			return nil
 		}
+		e := collectionFactory.PublishedRepoCollection().LoadComplete(published, collectionFactory)
+		if e != nil {
+			return e
+		}
 
 		for _, component := range published.Components() {
-			sl := deb.NewSplitRefList()
+			existingPackageRefs = existingPackageRefs.Merge(published.RefList(component), false, true)
-			e := collectionFactory.RefListCollection().LoadCompleteAndMigrate(sl, published.RefKey(component), reflistMigration)
-			if e != nil {
-				return e
-			}
-
-			existingPackageRefs = existingPackageRefs.Merge(sl, false, true)
-			existingBuckets.AddAllInRefList(sl)
-
 			if verbose {
 				description := fmt.Sprintf("published repository %s:%s/%s component %s",
 					published.Storage, published.Prefix, published.Distribution, component)
-				sl.ForEach(func(key []byte) error {
+				_ = published.RefList(component).ForEach(func(key []byte) error {
 					packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
 					return nil
 				})
@@ -178,29 +170,11 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 
 	collectionFactory.Flush()
 
-	err = reflistMigration.Flush()
-	if err != nil {
-		return err
-	}
-
-	if verbose {
-		if stats := reflistMigration.Stats(); stats.Reflists > 0 {
-			if !dryRun {
-				context.Progress().ColoredPrintf("@{w!}Split %d reflist(s) into %d bucket(s) (%d segment(s))@|",
-					stats.Reflists, stats.Buckets, stats.Segments)
-			} else {
-				context.Progress().ColoredPrintf(
-					"@{y!}Skipped splitting %d reflist(s) into %d bucket(s) (%d segment(s)), as -dry-run has been requested.@|",
-					stats.Reflists, stats.Buckets, stats.Segments)
-			}
-		}
-	}
-
 	// ... and compare it to the list of all packages
 	context.Progress().ColoredPrintf("@{w!}Loading list of all packages...@|")
 	allPackageRefs := collectionFactory.PackageCollection().AllPackageRefs()
 
-	toDelete := allPackageRefs.Subtract(existingPackageRefs.Flatten())
+	toDelete := allPackageRefs.Subtract(existingPackageRefs)
 
 	// delete packages that are no longer referenced
 	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced packages (%d)...@|", toDelete.Len())
@@ -238,32 +212,6 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	bucketsToDelete, err := collectionFactory.RefListCollection().AllBucketDigests()
-	if err != nil {
-		return err
-	}
-
-	bucketsToDelete.RemoveAll(existingBuckets)
-
-	context.Progress().ColoredPrintf("@{r!}Deleting unreferenced reflist buckets (%d)...@|", bucketsToDelete.Len())
-	if bucketsToDelete.Len() > 0 {
-		if !dryRun {
-			batch := db.CreateBatch()
-			err := bucketsToDelete.ForEach(func(digest []byte) error {
-				return collectionFactory.RefListCollection().UnsafeDropBucket(digest, batch)
-			})
-			if err != nil {
-				return err
-			}
-
-			if err := batch.Write(); err != nil {
-				return err
-			}
-		} else {
-			context.Progress().ColoredPrintf("@{y!}Skipped reflist deletion, as -dry-run has been requested.@|")
-		}
-	}
-
 	collectionFactory.Flush()
 
 	// now, build a list of files that should be present in Repository (package pool)
@@ -298,6 +246,7 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 		return err
 	}
 
+	referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
 	sort.Strings(referencedFiles)
 	context.Progress().ShutdownBar()
 
@@ -353,9 +302,9 @@ func aptlyDbCleanup(cmd *commander.Command, args []string) error {
 	return err
 }
 
-func makeCmdDbCleanup() *commander.Command {
+func makeCmdDBCleanup() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbCleanup,
+		Run:       aptlyDBCleanup,
 		UsageLine: "cleanup",
 		Short:     "cleanup DB and package pool",
 		Long: `

cmd/db_recover.go

@@ -1,13 +1,16 @@
 package cmd
 
 import (
+	"fmt"
+
+	"github.com/aptly-dev/aptly/deb"
 	"github.com/smira/commander"
 
 	"github.com/aptly-dev/aptly/database/goleveldb"
 )
 
 // aptly db recover
-func aptlyDbRecover(cmd *commander.Command, args []string) error {
+func aptlyDBRecover(cmd *commander.Command, args []string) error {
 	var err error
 
 	if len(args) != 0 {
@@ -16,14 +19,19 @@ func aptlyDbRecover(cmd *commander.Command, args []string) error {
 	}
 
 	context.Progress().Printf("Recovering database...\n")
-	err = goleveldb.RecoverDB(context.DBPath())
+	if err = goleveldb.RecoverDB(context.DBPath()); err != nil {
+		return err
+	}
+
+	context.Progress().Printf("Checking database integrity...\n")
+	err = checkIntegrity()
 
 	return err
 }
 
-func makeCmdDbRecover() *commander.Command {
+func makeCmdDBRecover() *commander.Command {
 	cmd := &commander.Command{
-		Run:       aptlyDbRecover,
+		Run:       aptlyDBRecover,
 		UsageLine: "recover",
 		Short:     "recover DB after crash",
 		Long: `
@@ -38,3 +46,36 @@ Example:
 
 	return cmd
 }
+
+func checkIntegrity() error {
+	return context.NewCollectionFactory().LocalRepoCollection().ForEach(checkRepo)
+}
+
+func checkRepo(repo *deb.LocalRepo) error {
+	collectionFactory := context.NewCollectionFactory()
+	repos := collectionFactory.LocalRepoCollection()
+
+	err := repos.LoadComplete(repo)
+	if err != nil {
+		return fmt.Errorf("load complete repo %q: %s", repo.Name, err)
+	}
+
+	dangling, err := deb.FindDanglingReferences(repo.RefList(), collectionFactory.PackageCollection())
+	if err != nil {
+		return fmt.Errorf("find dangling references: %w", err)
+	}
+
+	if len(dangling.Refs) > 0 {
+		for _, ref := range dangling.Refs {
+			context.Progress().Printf("Removing dangling database reference %q\n", ref)
+		}
+
+		repo.UpdateRefList(repo.RefList().Subtract(dangling))
+
+		if err = repos.Update(repo); err != nil {
+			return fmt.Errorf("update repo: %w", err)
+		}
+	}
+
+	return nil
+}

cmd/graph.go

@@ -38,8 +38,8 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	tempfile.Close()
+	_ = tempfile.Close()
-	os.Remove(tempfile.Name())
+	_ = os.Remove(tempfile.Name())
 
 	format := context.Flags().Lookup("format").Value.String()
 	output := context.Flags().Lookup("output").Value.String()

cmd/mirror.go

@@ -20,7 +20,7 @@ func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
 		verifier.AddKeyring(keyRing)
 	}
 
-	err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
+	err := verifier.InitKeyring(!ignoreSignatures) // be verbose only if verifying signatures is requested
 	if err != nil {
 		return nil, err
 	}

cmd/mirror_create.go

@@ -20,6 +20,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
 	downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
 	downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
+	downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
 	ignoreSignatures := context.Config().GpgDisableVerify
 	if context.Flags().IsSet("ignore-signatures") {
 		ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
@@ -41,7 +42,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
-		downloadSources, downloadUdebs, downloadInstaller)
+		downloadSources, downloadUdebs, downloadInstaller, downloadAppStream)
 	if err != nil {
 		return fmt.Errorf("unable to create mirror: %s", err)
 	}
@@ -69,7 +70,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	err = collectionFactory.RemoteRepoCollection().Add(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add mirror: %s", err)
 	}
@@ -100,6 +101,7 @@ Example:
 	}
 
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_edit.go

@@ -35,6 +35,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 			repo.Filter = flag.Value.String() // allows file/stdin with @
 		case "filter-with-deps":
 			repo.FilterWithDeps = flag.Value.Get().(bool)
+		case "with-appstream":
+			repo.DownloadAppStream = flag.Value.Get().(bool)
 		case "with-installer":
 			repo.DownloadInstaller = flag.Value.Get().(bool)
 		case "with-sources":
@@ -53,6 +55,10 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
 	}
 
+	if repo.IsFlat() && repo.DownloadAppStream {
+		return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
+	}
+
 	if repo.Filter != "" {
 		_, err = query.Parse(repo.Filter)
 		if err != nil {
@@ -78,7 +84,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -107,6 +113,7 @@ Example:
 	AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
 	cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
+	cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
 	cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
 	cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
 	cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")

cmd/mirror_list.go

@@ -32,7 +32,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 
 	repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
 	i := 0
-	collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
@@ -42,7 +42,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -70,13 +70,13 @@ func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
 
 	repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
+	_ = context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
 		repos[i] = repo
 		i++
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/mirror_rename.go

@@ -37,7 +37,7 @@ func aptlyMirrorRename(cmd *commander.Command, args []string) error {
 	}
 
 	repo.Name = newName
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/mirror_show.go

@@ -38,7 +38,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -61,6 +61,11 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		downloadUdebs = Yes
 	}
 	fmt.Printf("Download .udebs: %s\n", downloadUdebs)
+	downloadAppStream := No
+	if repo.DownloadAppStream {
+		downloadAppStream = Yes
+	}
+	fmt.Printf("Download AppStream: %s\n", downloadAppStream)
 	if repo.Filter != "" {
 		fmt.Printf("Filter: %s\n", repo.Filter)
 		filterWithDeps := No
@@ -86,7 +91,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
 		if repo.LastDownloadDate.IsZero() {
 			fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
 		} else {
-			ListPackagesRefList(repo.RefList(), collectionFactory)
+			_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 		}
 	}
 
@@ -98,13 +103,12 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 
 	name := args[0]
 
-	collectionFactory := context.NewCollectionFactory()
+	repo, err := context.NewCollectionFactory().RemoteRepoCollection().ByName(name)
-	repo, err := collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -114,13 +118,13 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if repo.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err != nil {
 				return fmt.Errorf("unable to get package list: %s", err)
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				repo.Packages = append(repo.Packages, p.GetFullName())
 				return nil
 			})

cmd/mirror_update.go

@@ -29,7 +29,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
-	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -64,6 +64,15 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: %s", err)
 	}
 
+	if repo.DownloadAppStream && !repo.IsFlat() {
+		context.Progress().Printf("Downloading AppStream metadata...\n")
+		err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
+			context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
+		if err != nil {
+			return fmt.Errorf("unable to update: %s", err)
+		}
+	}
+
 	if repo.Filter != "" {
 		context.Progress().Printf("Applying filter...\n")
 		var filterQuery deb.PackageQuery
@@ -87,10 +96,11 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 	)
 
 	skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
+	latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
 
 	context.Progress().Printf("Building download queue...\n")
 	queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-		collectionFactory.ChecksumCollection(nil), skipExistingPackages)
+		collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
 
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
@@ -101,12 +111,12 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		err = context.ReOpenDatabase()
 		if err == nil {
 			repo.MarkAsIdle()
-			collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+			_ = collectionFactory.RemoteRepoCollection().Update(repo)
 		}
 	}()
 
 	repo.MarkAsUpdating()
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -173,7 +183,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 						file, e = os.CreateTemp("", task.File.Filename)
 						if e == nil {
 							task.TempDownPath = file.Name()
-							file.Close()
+							_ = file.Close()
 						}
 					}
 					if e != nil {
@@ -261,8 +271,8 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to update: download errors:\n  %s", strings.Join(errors, "\n  "))
 	}
 
-	repo.FinalizeDownload(collectionFactory, context.Progress())
+	_ = repo.FinalizeDownload(collectionFactory, context.Progress())
-	err = collectionFactory.RemoteRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.RemoteRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to update: %s", err)
 	}
@@ -292,6 +302,7 @@ Example:
 	cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
 	cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
 	cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
+	cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
 	cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
 	cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
 	cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")

cmd/package_search.go

@@ -40,7 +40,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }

cmd/package_show.go

@@ -14,7 +14,7 @@ import (
 
 func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory) (err error) {
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
-		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -30,7 +30,7 @@ func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory)
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
@@ -46,7 +46,7 @@ func printReferencesTo(p *deb.Package, collectionFactory *deb.CollectionFactory)
 	}
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *deb.Snapshot) error {
-		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -84,8 +84,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 	result := q.Query(collectionFactory.PackageCollection())
 
 	err = result.ForEach(func(p *deb.Package) error {
-		p.Stanza().WriteTo(w, p.IsSource, false, false)
+		_ = p.Stanza().WriteTo(w, p.IsSource, false, false)
-		w.Flush()
+		_ = w.Flush()
 		fmt.Printf("\n")
 
 		if withFiles {
@@ -109,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
 
 		if withReferences {
 			fmt.Printf("References to package:\n")
-			printReferencesTo(p, collectionFactory)
+			_ = printReferencesTo(p, collectionFactory)
 			fmt.Printf("\n")
 		}
 

cmd/publish.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+
+	var gpgKeys []string
+
+	// CLI args have priority over config
+	cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
+	if len(cliKeys) > 0 {
+		gpgKeys = cliKeys
+	} else if len(context.Config().GpgKeys) > 0 {
+		gpgKeys = context.Config().GpgKeys
+	}
+
+	for _, gpgKey := range gpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 
 }
 
+type gpgKeyFlag struct {
+	gpgKeys []string
+}
+
+func (k *gpgKeyFlag) Set(value string) error {
+	k.gpgKeys = append(k.gpgKeys, value)
+	return nil
+}
+
+func (k *gpgKeyFlag) Get() interface{} {
+	return k.gpgKeys
+}
+
+func (k *gpgKeyFlag) String() string {
+	return strings.Join(k.gpgKeys, ",")
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_list.go

@@ -53,7 +53,7 @@ func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(published)
 
@@ -99,7 +99,7 @@ func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
 		return fmt.Errorf("unable to load list of repos: %s", err)
 	}
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].GetPath() < repos[j].GetPath()

cmd/publish_repo.go

@@ -34,7 +34,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -51,7 +51,9 @@ Example:
 	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
 	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+	cmd.Flag.String("version", "", "version of the release")
 
 	return cmd
 }

cmd/publish_snapshot.go

@@ -49,7 +49,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+			err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -85,7 +85,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
 
-			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo, collectionFactory.RefListCollection())
+			err = collectionFactory.LocalRepoCollection().LoadComplete(localRepo)
 			if err != nil {
 				return fmt.Errorf("unable to publish: %s", err)
 			}
@@ -150,13 +150,21 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
 	if context.Flags().IsSet("multi-dist") {
 		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
 	duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
 	if duplicate != nil {
-		collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+		_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
 		return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 	}
 
@@ -175,7 +183,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Add(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Add(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -230,7 +238,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -247,6 +255,8 @@ Example:
 	cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
 	cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
+	cmd.Flag.String("version", "", "version of the release")
 	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 
 	return cmd

cmd/publish_switch.go

@@ -72,7 +72,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
 
-		err = snapshotCollection.LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = snapshotCollection.LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to switch: %s", err)
 		}
@@ -99,6 +99,14 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
 	if context.Flags().IsSet("multi-dist") {
 		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
 	}
@@ -108,7 +116,7 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Update(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -151,7 +159,7 @@ This command would switch published repository (with one component) named ppa/wh
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -162,6 +170,8 @@ This command would switch published repository (with one component) named ppa/wh
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
+	cmd.Flag.String("version", "", "version of the release")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
 

cmd/publish_update.go

@@ -60,16 +60,32 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
 		published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("signed-by") {
+		published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
+	}
+
+	if context.Flags().IsSet("origin") {
+		published.Origin = context.Flags().Lookup("origin").Value.String()
+	}
+
+	if context.Flags().IsSet("label") {
+		published.Label = context.Flags().Lookup("label").Value.String()
+	}
+
 	if context.Flags().IsSet("multi-dist") {
 		published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
 	}
 
+	if context.Flags().IsSet("version") {
+		published.Version = context.Flags().Lookup("version").Value.String()
+	}
+
 	err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
 	if err != nil {
 		return fmt.Errorf("unable to publish: %s", err)
 	}
 
-	err = collectionFactory.PublishedRepoCollection().Update(published, collectionFactory.RefListCollection())
+	err = collectionFactory.PublishedRepoCollection().Update(published)
 	if err != nil {
 		return fmt.Errorf("unable to save to DB: %s", err)
 	}
@@ -115,7 +131,7 @@ Example:
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -125,8 +141,12 @@ Example:
 	cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
 	cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
 	cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
+	cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
 	cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
 	cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+    cmd.Flag.String("origin", "", "overwrite origin name to publish")
+    cmd.Flag.String("label", "", "overwrite label to publish")
+    cmd.Flag.String("version", "", "version of the release")
 
 	return cmd
 }

cmd/repo_add.go

@@ -28,7 +28,7 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to add: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add: %s", err)
 	}
@@ -58,9 +58,9 @@ func aptlyRepoAdd(cmd *commander.Command, args []string) error {
 
 	processedFiles = append(processedFiles, otherFiles...)
 
-	repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+	repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to save: %s", err)
 	}

cmd/repo_create.go

@@ -36,7 +36,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to load source snapshot: %s", err)
 		}
@@ -44,7 +44,7 @@ func aptlyRepoCreate(cmd *commander.Command, args []string) error {
 		repo.UpdateRefList(snapshot.RefList())
 	}
 
-	err = collectionFactory.LocalRepoCollection().Add(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Add(repo)
 	if err != nil {
 		return fmt.Errorf("unable to add local repo: %s", err)
 	}

cmd/repo_edit.go

@@ -22,7 +22,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}
@@ -53,7 +53,7 @@ func aptlyRepoEdit(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to edit: %s", err)
 	}

cmd/repo_include.go

@@ -67,7 +67,7 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
 	_, failedFiles2, err = deb.ImportChangesFiles(
 		changesFiles, reporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles, verifier, repoTemplate,
 		context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-		collectionFactory.RefListCollection(), context.PackagePool(), collectionFactory.ChecksumCollection,
+		context.PackagePool(), collectionFactory.ChecksumCollection,
 		uploaders, query.Parse)
 	failedFiles = append(failedFiles, failedFiles2...)
 

cmd/repo_list.go

@@ -32,11 +32,11 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 	collectionFactory := context.NewCollectionFactory()
 	repos := make([]string, collectionFactory.LocalRepoCollection().Len())
 	i := 0
-	collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
 		if raw {
 			repos[i] = repo.Name
 		} else {
-			e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+			e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 			if e != nil {
 				return e
 			}
@@ -47,7 +47,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Strings(repos)
 
@@ -76,9 +76,8 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 
 	repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
 	i := 0
-	context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
+	_ = context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
-		collectionFactory := context.NewCollectionFactory()
+		e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
 		if e != nil {
 			return e
 		}
@@ -88,7 +87,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
 		return nil
 	})
 
-	context.CloseDatabase()
+	_ = context.CloseDatabase()
 
 	sort.Slice(repos, func(i, j int) bool {
 		return repos[i].Name < repos[j].Name

cmd/repo_move.go

@@ -25,13 +25,13 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
 	if err != nil {
 		return fmt.Errorf("unable to %s: %s", command, err)
 	}
 
 	var (
-		srcRefList *deb.SplitRefList
+		srcRefList *deb.PackageRefList
 		srcRepo    *deb.LocalRepo
 	)
 
@@ -45,7 +45,7 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: source and destination are the same", command)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
@@ -59,12 +59,12 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(srcRemoteRepo)
 		if err != nil {
 			return fmt.Errorf("unable to %s: %s", command, err)
 		}
 
-		if srcRemoteRepo.RefList().Len() == 0 {
+		if srcRemoteRepo.RefList() == nil {
 			return fmt.Errorf("unable to %s: mirror not updated", command)
 		}
 
@@ -161,17 +161,17 @@ func aptlyRepoMoveCopyImport(cmd *commander.Command, args []string) error {
 	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		dstRepo.UpdateRefList(deb.NewSplitRefListFromPackageList(dstList))
+		dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-		err = collectionFactory.LocalRepoCollection().Update(dstRepo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(dstRepo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}
 
 		if command == "move" { // nolint: goconst
-			srcRepo.UpdateRefList(deb.NewSplitRefListFromPackageList(srcList))
+			srcRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(srcList))
 
-			err = collectionFactory.LocalRepoCollection().Update(srcRepo, collectionFactory.RefListCollection())
+			err = collectionFactory.LocalRepoCollection().Update(srcRepo)
 			if err != nil {
 				return fmt.Errorf("unable to save: %s", err)
 			}

cmd/repo_remove.go

@@ -24,7 +24,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to remove: %s", err)
 	}
 
-	toRemove.ForEach(func(p *deb.Package) error {
+	_ = toRemove.ForEach(func(p *deb.Package) error {
 		list.Remove(p)
 		context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
 		return nil
@@ -63,9 +63,9 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
 	if context.Flags().Lookup("dry-run").Value.Get().(bool) {
 		context.Progress().Printf("\nChanges not saved, as dry run has been requested.\n")
 	} else {
-		repo.UpdateRefList(deb.NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().Update(repo)
 		if err != nil {
 			return fmt.Errorf("unable to save: %s", err)
 		}

cmd/repo_rename.go

@@ -32,7 +32,7 @@ func aptlyRepoRename(cmd *commander.Command, args []string) error {
 	}
 
 	repo.Name = newName
-	err = collectionFactory.LocalRepoCollection().Update(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().Update(repo)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/repo_show.go

@@ -36,7 +36,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -52,7 +52,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(repo.RefList(), collectionFactory)
+		_ = ListPackagesRefList(repo.RefList(), collectionFactory)
 	}
 
 	return err
@@ -63,13 +63,12 @@ func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 
 	name := args[0]
 
-	collectionFactory := context.NewCollectionFactory()
+	repo, err := context.NewCollectionFactory().LocalRepoCollection().ByName(name)
-	repo, err := collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -80,7 +79,7 @@ func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if repo.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(repo.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err == nil {
 				packageList = list.FullNames()
 			}

cmd/snapshot_create.go

@@ -30,7 +30,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -50,7 +50,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -71,7 +71,7 @@ func aptlySnapshotCreate(cmd *commander.Command, args []string) error {
 		return commander.ErrCommandError
 	}
 
-	err = collectionFactory.SnapshotCollection().Add(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to add snapshot: %s", err)
 	}

cmd/snapshot_diff.go

@@ -23,7 +23,7 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotA)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot A: %s", err)
 	}
@@ -34,13 +34,13 @@ func aptlySnapshotDiff(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshotB)
 	if err != nil {
 		return fmt.Errorf("unable to load snapshot B: %s", err)
 	}
 
 	// Calculate diff
-	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection(), nil)
+	diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
 	if err != nil {
 		return fmt.Errorf("unable to calculate diff: %s", err)
 	}

cmd/snapshot_filter.go

@@ -27,7 +27,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(source, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to filter: %s", err)
 	}
@@ -87,7 +87,7 @@ func aptlySnapshotFilter(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromPackageList(args[1], []*deb.Snapshot{source}, result,
 		fmt.Sprintf("Filtered '%s', query was: '%s'", source.Name, strings.Join(args[2:], " ")))
 
-	err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_list.go

@@ -33,7 +33,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
 	collection := collectionFactory.SnapshotCollection()
 
 	if raw {
-		collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+		_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 			fmt.Printf("%s\n", snapshot.Name)
 			return nil
 		})
@@ -68,7 +68,7 @@ func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
 
 	jsonSnapshots := make([]*deb.Snapshot, collection.Len())
 	i := 0
-	collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
+	_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
 		jsonSnapshots[i] = snapshot
 		i++
 		return nil

cmd/snapshot_merge.go

@@ -24,7 +24,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i], collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(sources[i])
 		if err != nil {
 			return fmt.Errorf("unable to load snapshot: %s", err)
 		}
@@ -57,7 +57,7 @@ func aptlySnapshotMerge(cmd *commander.Command, args []string) error {
 	destination := deb.NewSnapshotFromRefList(args[0], sources, result,
 		fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-	err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Add(destination)
 	if err != nil {
 		return fmt.Errorf("unable to create snapshot: %s", err)
 	}

cmd/snapshot_pull.go

@@ -29,7 +29,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -40,7 +40,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(source, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(source)
 	if err != nil {
 		return fmt.Errorf("unable to pull: %s", err)
 	}
@@ -116,7 +116,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 	alreadySeen := map[string]bool{}
 
-	result.ForEachIndexed(func(pkg *deb.Package) error {
+	_ = result.ForEachIndexed(func(pkg *deb.Package) error {
 		key := pkg.Architecture + "_" + pkg.Name
 		_, seen := alreadySeen[key]
 
@@ -132,7 +132,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 
 		// If !allMatches, add only first matching name-arch package
 		if !seen || allMatches {
-			packageList.Add(pkg)
+			_ = packageList.Add(pkg)
 			context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
 		}
 
@@ -149,7 +149,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
 		destination := deb.NewSnapshotFromPackageList(args[2], []*deb.Snapshot{snapshot, source}, packageList,
 			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", snapshot.Name, source.Name, strings.Join(args[3:], " ")))
 
-		err = collectionFactory.SnapshotCollection().Add(destination, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().Add(destination)
 		if err != nil {
 			return fmt.Errorf("unable to create snapshot: %s", err)
 		}

cmd/snapshot_rename.go

@@ -32,7 +32,7 @@ func aptlySnapshotRename(cmd *commander.Command, args []string) error {
 	}
 
 	snapshot.Name = newName
-	err = collectionFactory.SnapshotCollection().Update(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().Update(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to rename: %s", err)
 	}

cmd/snapshot_search.go

@@ -25,7 +25,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	command := cmd.Parent.Name()
 	collectionFactory := context.NewCollectionFactory()
 
-	var reflist *deb.SplitRefList
+	var reflist *deb.PackageRefList
 
 	if command == "snapshot" { // nolint: goconst
 		var snapshot *deb.Snapshot
@@ -34,7 +34,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -47,7 +47,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -60,7 +60,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 			return fmt.Errorf("unable to search: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		err = collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if err != nil {
 			return fmt.Errorf("unable to search: %s", err)
 		}
@@ -123,7 +123,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
 	}
 
 	format := context.Flags().Lookup("format").Value.String()
-	PrintPackageList(result, format, "")
+	_ = PrintPackageList(result, format, "")
 
 	return err
 }

cmd/snapshot_show.go

@@ -35,7 +35,7 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -79,24 +79,23 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
 
 	withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
 	if withPackages {
-		ListPackagesRefList(snapshot.RefList(), collectionFactory)
+		_ = ListPackagesRefList(snapshot.RefList(), collectionFactory)
 	}
 
 	return err
 }
 
 func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
-	collectionFactory := context.NewCollectionFactory()
 	var err error
 
 	name := args[0]
 
-	snapshot, err := collectionFactory.SnapshotCollection().ByName(name)
+	snapshot, err := context.NewCollectionFactory().SnapshotCollection().ByName(name)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
 
-	err = collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+	err = context.NewCollectionFactory().SnapshotCollection().LoadComplete(snapshot)
 	if err != nil {
 		return fmt.Errorf("unable to show: %s", err)
 	}
@@ -106,14 +105,14 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 		for _, sourceID := range snapshot.SourceIDs {
 			if snapshot.SourceKind == deb.SourceSnapshot {
 				var source *deb.Snapshot
-				source, err = collectionFactory.SnapshotCollection().ByUUID(sourceID)
+				source, err = context.NewCollectionFactory().SnapshotCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
 				snapshot.Snapshots = append(snapshot.Snapshots, source)
 			} else if snapshot.SourceKind == deb.SourceLocalRepo {
 				var source *deb.LocalRepo
-				source, err = collectionFactory.LocalRepoCollection().ByUUID(sourceID)
+				source, err = context.NewCollectionFactory().LocalRepoCollection().ByUUID(sourceID)
 				if err != nil {
 					continue
 				}
@@ -134,13 +133,13 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
 	if withPackages {
 		if snapshot.RefList() != nil {
 			var list *deb.PackageList
-			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+			list, err = deb.NewPackageListFromRefList(snapshot.RefList(), context.NewCollectionFactory().PackageCollection(), context.Progress())
 			if err != nil {
 				return fmt.Errorf("unable to get package list: %s", err)
 			}
 
 			list.PrepareIndex()
-			list.ForEachIndexed(func(p *deb.Package) error {
+			_ = list.ForEachIndexed(func(p *deb.Package) error {
 				snapshot.Packages = append(snapshot.Packages, p.GetFullName())
 				return nil
 			})

cmd/snapshot_verify.go

@@ -23,7 +23,7 @@ func aptlySnapshotVerify(cmd *commander.Command, args []string) error {
 			return fmt.Errorf("unable to verify: %s", err)
 		}
 
-		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i], collectionFactory.RefListCollection())
+		err = collectionFactory.SnapshotCollection().LoadComplete(snapshots[i])
 		if err != nil {
 			return fmt.Errorf("unable to verify: %s", err)
 		}

cmd/task_run.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mattn/go-shellwords"
+	shellwords "github.com/mattn/go-shellwords"
 	"github.com/smira/commander"
 )
 
@@ -31,7 +31,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 		if err != nil {
 			return err
 		}
-		defer file.Close()
+		defer func() { _ = file.Close() }()
 
 		scanner := bufio.NewScanner(file)
 

completion.d/_aptly

@@ -185,6 +185,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
                         ;;
                     list)
@@ -211,6 +212,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             $keyring \
                             "-max-tries=[max download tries till process fails with download error]:number: " \
                             "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
+                            "-latest=[download only latest version of each package (per architecture)]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     rename)
@@ -223,6 +225,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
                             "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
                             "-with-sources=[download source packages in addition to binary packages]:$bool" \
                             "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
+                            "-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
                             "(-)2:mirror name:$mirrors"
                         ;;
                     search)

completion.d/aptly

@@ -22,34 +22,36 @@
 
 __aptly_mirror_list()
 {
-  aptly mirror list -raw
+  aptly ${aptly_global_opts[@]} mirror list -raw
 }
 
 __aptly_repo_list()
 {
-  aptly repo list -raw
+  aptly ${aptly_global_opts[@]} repo list -raw
 }
 
 __aptly_snapshot_list()
 {
-  aptly snapshot list -raw
+  aptly ${aptly_global_opts[@]} snapshot list -raw
 }
 
 __aptly_published_distributions()
 {
-  aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 2 | sort | uniq
 }
 
 __aptly_published_prefixes()
 {
-  aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 1 | sort | uniq
 }
 
 __aptly_prefixes_for_distribution()
 {
-  aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
+  aptly ${aptly_global_opts[@]} publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
 }
 
+
+
 _aptly()
 {
     cur="${COMP_WORDS[COMP_CWORD]}"
@@ -57,7 +59,12 @@ _aptly()
     prevprev="${COMP_WORDS[COMP_CWORD-2]}"
 
     commands="api config db graph mirror package publish repo serve snapshot task version"
-    options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
+
+    options="-architectures -config -db-open-attempts -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider"
+    options_without_arg="-dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve"
+    options_with_arg="-architectures -db-open-attempts -gpg-provider"
+    options_with_path_arg="-config"
+
     db_subcommands="cleanup recover"
     mirror_subcommands="create drop edit show list rename search update"
     publish_subcommands="drop list repo snapshot switch update source"
@@ -69,12 +76,41 @@ _aptly()
     config_subcommands="show"
     api_subcommands="serve"
 
-    local cmd subcmd numargs numoptions i
+    local cmd subcmd numargs numoptions i aptly_global_opts
 
     numargs=0
     numoptions=0
 
+    for opt in "${options_with_path_arg[@]}"; do
+        [[ "$prev" == "$opt" ]] || continue
+        compopt -o filenames 2>/dev/null
+        _filedir
+        return 0
+    done
+
     for (( i=1; i < $COMP_CWORD; i++ )); do
+      word=${COMP_WORDS[i]}
+      if [[ "$word" == -*=* ]]; then
+        for o in "${options[@]}"; do
+          [[ ${word%%=*} == "$o" ]] && aptly_global_opts+=("$word")
+        done
+      else
+        for o in "${options_with_arg[@]}" ""${options_with_path_arg[@]}"" ; do
+          if [[ "$word" == "$o" ]]; then
+            if (( i + 1 < COMP_CWORD )); then
+                aptly_global_opts+=("$word" "${COMP_WORDS[i+1]}")
+            else
+                aptly_global_opts+=("$word")
+            fi
+            (( i++ ))
+            continue 2
+          fi
+        done
+      fi
+      for o in ${options_without_arg[@]}; do
+          [[ "$word" == "$o" ]] && aptly_global_opts+=("$word")
+      done
+
       if [[ -n "$cmd" ]]; then
         if [[ ! -n "$subcmd" ]]; then
           subcmd=${COMP_WORDS[i]}
@@ -167,7 +203,7 @@ _aptly()
           "create")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
                 return 0
               fi
             fi
@@ -175,7 +211,7 @@ _aptly()
           "edit")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
+                COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi
@@ -227,7 +263,7 @@ _aptly()
           "update")
             if [[ $numargs -eq 0 ]]; then
               if [[ "$cur" == -* ]]; then
-                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
+                COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur}))
               else
                 COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
               fi
@@ -339,7 +375,7 @@ _aptly()
                 if [[ "$cur" == -* ]]; then
                   COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
                 else
-                  comptopt -o filenames 2>/dev/null
+                  compopt -o filenames 2>/dev/null
                   COMPREPLY=($(compgen -f -- ${cur}))
                   return 0
                 fi

console/progress.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/aptly-dev/aptly/aptly"
+	"github.com/aptly-dev/aptly/utils"
 	"github.com/cheggaaa/pb"
 	"github.com/rs/zerolog/log"
 	"github.com/wsxiaoys/terminal/color"
@@ -78,7 +79,7 @@ func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
 	if p.bar != nil {
 		panic("bar already initialized")
 	}
-	if RunningOnTerminal() {
+	if utils.RunningOnTerminal() {
 		p.bar = pb.New(0)
 		p.bar.Total = count
 		p.bar.NotPrint = true
@@ -141,7 +142,7 @@ func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
 
 // ColoredPrintf does printf in colored way + newline
 func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
-	if RunningOnTerminal() {
+	if utils.RunningOnTerminal() {
 		p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
 	} else {
 		// stip color marks

console/progress_test.go

@@ -11,7 +11,7 @@ func Test(t *testing.T) {
 	TestingT(t)
 }
 
-type ProgressSuite struct {}
+type ProgressSuite struct{}
 
 var _ = Suite(&ProgressSuite{})
 

console/terminal.go

@@ -1,12 +0,0 @@
-package console
-
-import (
-	"syscall"
-
-	"golang.org/x/term"
-)
-
-// RunningOnTerminal checks whether stdout is terminal
-func RunningOnTerminal() bool {
-	return term.IsTerminal(syscall.Stdout)
-}

context/context.go

@@ -115,7 +115,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
-				utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
+				_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
 				err = utils.LoadConfig(homeLocation, &utils.Config)
 				if err != nil {
 					Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -123,6 +123,14 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			}
 		}
 
+		if utils.Config.LogFormat == "json" {
+			context.StructuredLogging(true)
+			utils.SetupJSONLogger(utils.Config.LogLevel, os.Stdout)
+		} else {
+			context.StructuredLogging(false)
+			utils.SetupDefaultLogger(utils.Config.LogLevel)
+		}
+
 		context.configLoaded = true
 
 	}
@@ -233,7 +241,7 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo
 		// If flag is defined prefer it to global setting
 		maxTries = maxTriesFlag.Value.Get().(int)
 	}
-	var downloader string = context.config().Downloader
+	var downloader = context.config().Downloader
 	downloaderFlag := context.flags.Lookup("downloader")
 	if downloaderFlag != nil {
 		downloader = downloaderFlag.Value.String()
@@ -295,8 +303,8 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 		switch context.config().DatabaseBackend.Type {
 		case "leveldb":
 			dbPath := filepath.Join(context.config().GetRootDir(), "db")
-			if len(context.config().DatabaseBackend.DbPath) != 0 {
+			if len(context.config().DatabaseBackend.DBPath) != 0 {
-				dbPath = context.config().DatabaseBackend.DbPath
+				dbPath = context.config().DatabaseBackend.DBPath
 			}
 			context.database, err = goleveldb.NewDB(dbPath)
 		case "etcd":
@@ -444,7 +452,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 		} else if strings.HasPrefix(name, "azure:") {
 			params, ok := context.config().AzurePublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:]))
+				Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:]))
 			}
 
 			var err error
@@ -589,17 +597,17 @@ func (context *AptlyContext) Shutdown() {
 
 	if aptly.EnableDebug {
 		if context.fileMemProfile != nil {
-			pprof.WriteHeapProfile(context.fileMemProfile)
+			_ = pprof.WriteHeapProfile(context.fileMemProfile)
-			context.fileMemProfile.Close()
+			_ = context.fileMemProfile.Close()
 			context.fileMemProfile = nil
 		}
 		if context.fileCPUProfile != nil {
 			pprof.StopCPUProfile()
-			context.fileCPUProfile.Close()
+			_ = context.fileCPUProfile.Close()
 			context.fileCPUProfile = nil
 		}
 		if context.fileMemProfile != nil {
-			context.fileMemProfile.Close()
+			_ = context.fileMemProfile.Close()
 			context.fileMemProfile = nil
 		}
 	}
@@ -607,7 +615,7 @@ func (context *AptlyContext) Shutdown() {
 		context.taskList.Stop()
 	}
 	if context.database != nil {
-		context.database.Close()
+		_ = context.database.Close()
 		context.database = nil
 	}
 	if context.downloader != nil {
@@ -652,7 +660,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 			if err != nil {
 				return nil, err
 			}
-			pprof.StartCPUProfile(context.fileCPUProfile)
+			_ = pprof.StartCPUProfile(context.fileCPUProfile)
 		}
 
 		memprofile := flags.Lookup("memprofile").Value.String()
@@ -672,7 +680,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 				return nil, err
 			}
 
-			context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
+			_, _ = context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
 
 			go func() {
 				var stats runtime.MemStats
@@ -682,7 +690,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
 				for {
 					runtime.ReadMemStats(&stats)
 					if context.fileMemStats != nil {
-						context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
+						_, _ = context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
 							(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
 						time.Sleep(interval)
 					} else {

database/database.go

@@ -48,8 +48,6 @@ type Storage interface {
 
 	CreateTemporary() (Storage, error)
 
-	GetRecommendedMaxKVSize() int
-
 	Open() error
 	Close() error
 	CompactDB() error

database/etcddb/database_test.go

@@ -14,8 +14,7 @@ func Test(t *testing.T) {
 }
 
 type EtcDDBSuite struct {
-	url string
+	db database.Storage
-	db  database.Storage
 }
 
 var _ = Suite(&EtcDDBSuite{})
@@ -67,17 +66,17 @@ func (s *EtcDDBSuite) TestDelete(c *C) {
 func (s *EtcDDBSuite) TestByPrefix(c *C) {
 	//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
@@ -109,7 +108,7 @@ func (s *EtcDDBSuite) TestHasPrefix(c *C) {
 	//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
 	//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
 
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -124,15 +123,17 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 		value2 = []byte("value2")
 	)
 	transaction, err := s.db.OpenTransaction()
+	c.Assert(err, IsNil)
 
 	err = s.db.Put(key, value)
 	c.Assert(err, IsNil)
 
 	c.Assert(err, IsNil)
-	transaction.Put(key2, value2)
+	_ = transaction.Put(key2, value2)
 	v, err := s.db.Get(key)
+	c.Assert(err, IsNil)
 	c.Check(v, DeepEquals, value)
-        err = transaction.Delete(key)
+	err = transaction.Delete(key)
 	c.Assert(err, IsNil)
 
 	_, err = transaction.Get(key2)
@@ -155,4 +156,3 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
 	_, err = transaction.Get(key)
 	c.Assert(err, NotNil)
 }
-

database/etcddb/storage.go

@@ -145,7 +145,7 @@ func (s *EtcDStorage) Close() error {
 	return err
 }
 
-// Reopen tries to open (re-open) the database
+// Open returns the database
 func (s *EtcDStorage) Open() error {
 	if s.db != nil {
 		return nil

database/etcddb/transaction.go

@@ -67,8 +67,7 @@ func (t *transaction) Commit() (err error) {
 // Discard is safe to call after Commit(), it would be no-op
 func (t *transaction) Discard() {
 	t.ops = []clientv3.Op{}
-	t.tmpdb.Drop()
+	_ = t.tmpdb.Drop()
-	return
 }
 
 // transaction should implement database.Transaction

database/goleveldb/database.go

@@ -9,13 +9,10 @@ import (
 	"github.com/aptly-dev/aptly/database"
 )
 
-const blockSize = 4 * 1024
-
 func internalOpen(path string, throttleCompaction bool) (*leveldb.DB, error) {
 	o := &opt.Options{
 		Filter:                 filter.NewBloomFilter(10),
 		OpenFilesCacheCapacity: 256,
-		BlockSize:              blockSize,
 	}
 
 	if throttleCompaction {
@@ -54,8 +51,8 @@ func RecoverDB(path string) error {
 		return err
 	}
 
-	db.Close()
+	_ = db.Close()
-	stor.Close()
+	_ = stor.Close()
 
 	return nil
 }

database/goleveldb/database_test.go

@@ -119,17 +119,17 @@ func (s *LevelDBSuite) TestDelete(c *C) {
 func (s *LevelDBSuite) TestByPrefix(c *C) {
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
+	_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
-	s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
+	_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
 	c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
 	c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
 
@@ -161,7 +161,7 @@ func (s *LevelDBSuite) TestHasPrefix(c *C) {
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
 
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
 
 	c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
 	c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -180,8 +180,8 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 	c.Assert(err, IsNil)
 
 	batch := s.db.CreateBatch()
-	batch.Put(key2, value2)
+	_ = batch.Put(key2, value2)
-	batch.Delete(key)
+	_ = batch.Delete(key)
 
 	v, err := s.db.Get(key)
 	c.Check(err, IsNil)
@@ -202,9 +202,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
 }
 
 func (s *LevelDBSuite) TestCompactDB(c *C) {
-	s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
+	_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
-	s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
+	_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
-	s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
+	_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
 
 	c.Check(s.db.CompactDB(), IsNil)
 }

database/goleveldb/storage.go

@@ -16,17 +16,6 @@ type storage struct {
 	db   *leveldb.DB
 }
 
-func (s *storage) GetRecommendedMaxKVSize() int {
-	// The block size configured is not actually a *set* block size, but rather a
-	// *minimum*. LevelDB only checks if a block is full after a new key/value pair is
-	// written, meaning that blocks will tend to overflow a bit.
-	// Therefore, using the default block size as the max value size will ensure
-	// that a new block will only contain a single value and that the size will
-	// only ever be as large as around double the block size (if the block was
-	// nearly full before the new items were added).
-	return blockSize
-}
-
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *storage) CreateTemporary() (database.Storage, error) {
 	tempdir, err := os.MkdirTemp("", "aptly")

deb/changes.go

@@ -60,14 +60,14 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 	if err != nil {
 		return err
 	}
-	defer input.Close()
+	defer func() { _ = input.Close() }()
 
 	isClearSigned, err := verifier.IsClearSigned(input)
 	if err != nil {
 		return err
 	}
 
-	input.Seek(0, 0)
+	_, _ = input.Seek(0, 0)
 
 	if !isClearSigned && !acceptUnsigned {
 		return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
@@ -79,7 +79,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		if err != nil {
 			return err
 		}
-		input.Seek(0, 0)
+		_, _ = input.Seek(0, 0)
 
 		c.SignatureKeys = keyInfo.GoodKeys
 	}
@@ -91,7 +91,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
 		if err != nil {
 			return err
 		}
-		defer text.Close()
+		defer func() { _ = text.Close() }()
 	} else {
 		text = input
 	}
@@ -291,8 +291,7 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
 // ImportChangesFiles imports referenced files in changes files into local repository
 func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, acceptUnsigned, ignoreSignatures, forceReplace, noRemoveFiles bool,
 	verifier pgp.Verifier, repoTemplate *template.Template, progress aptly.Progress, localRepoCollection *LocalRepoCollection, packageCollection *PackageCollection,
-	reflistCollection *RefListCollection, pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders,
+	pool aptly.PackagePool, checksumStorageProvider aptly.ChecksumStorageProvider, uploaders *Uploaders, parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
-	parseQuery parseQuery) (processedFiles []string, failedFiles []string, err error) {
 
 	for _, path := range changesFiles {
 		var changes *Changes
@@ -308,7 +307,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -316,7 +315,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -335,7 +334,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 		if err != nil {
 			failedFiles = append(failedFiles, path)
 			reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
-			changes.Cleanup()
+			_ = changes.Cleanup()
 			continue
 		}
 
@@ -355,12 +354,12 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 				failedFiles = append(failedFiles, path)
 				reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
 					changes.ChangesName, changes.SignatureKeys, err)
-				changes.Cleanup()
+				_ = changes.Cleanup()
 				continue
 			}
 		}
 
-		err = localRepoCollection.LoadComplete(repo, reflistCollection)
+		err = localRepoCollection.LoadComplete(repo)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to load repo: %s", err)
 		}
@@ -383,9 +382,9 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
 			return nil, nil, fmt.Errorf("unable to import package files: %s", err)
 		}
 
-		repo.UpdateRefList(NewSplitRefListFromPackageList(list))
+		repo.UpdateRefList(NewPackageRefListFromPackageList(list))
 
-		err = localRepoCollection.Update(repo, reflistCollection)
+		err = localRepoCollection.Update(repo)
 		if err != nil {
 			return nil, nil, fmt.Errorf("unable to save: %s", err)
 		}

deb/changes_test.go

@@ -21,7 +21,6 @@ type ChangesSuite struct {
 	db                  database.Storage
 	localRepoCollection *LocalRepoCollection
 	packageCollection   *PackageCollection
-	reflistCollection   *RefListCollection
 	packagePool         aptly.PackagePool
 	checksumStorage     aptly.ChecksumStorage
 	progress            aptly.Progress
@@ -43,7 +42,6 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
 	s.localRepoCollection = NewLocalRepoCollection(s.db)
 	s.packageCollection = NewPackageCollection(s.db)
-	s.reflistCollection = NewRefListCollection(s.db)
 
 	s.checksumStorage = files.NewMockChecksumStorage()
 	s.packagePool = files.NewPackagePool(s.Dir, false)
@@ -53,7 +51,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
 
 func (s *ChangesSuite) TearDownTest(c *C) {
 	s.progress.Shutdown()
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -90,7 +88,7 @@ func (s *ChangesSuite) TestCollectChangesFiles(c *C) {
 
 func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	repo := NewLocalRepo("test", "Test Comment")
-	c.Assert(s.localRepoCollection.Add(repo, s.reflistCollection), IsNil)
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
 
 	origFailedFiles := []string{
 		"testdata/changes/calamares.changes",
@@ -110,13 +108,13 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 
 	for _, path := range origFailedFiles {
 		filename := filepath.Join(s.Dir, filepath.Base(path))
-		utils.CopyFile(path, filename)
+		_ = utils.CopyFile(path, filename)
 		expectedFailedFiles = append(expectedFailedFiles, filename)
 	}
 
 	for _, path := range origProcessedFiles {
 		filename := filepath.Join(s.Dir, filepath.Base(path))
-		utils.CopyFile(path, filename)
+		_ = utils.CopyFile(path, filename)
 		expectedProcessedFiles = append(expectedProcessedFiles, filename)
 	}
 
@@ -126,8 +124,7 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 	processedFiles, failedFiles, err := ImportChangesFiles(
 		append(changesFiles, "testdata/changes/notexistent.changes"),
 		s.Reporter, true, true, false, false, &NullVerifier{},
-		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.reflistCollection, s.packagePool,
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
-		func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, DeepEquals, append(expectedFailedFiles, "testdata/changes/notexistent.changes"))
@@ -136,7 +133,7 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
 
 func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
 	repo := NewLocalRepo("test", "Test Comment")
-	c.Assert(s.localRepoCollection.Add(repo, s.reflistCollection), IsNil)
+	c.Assert(s.localRepoCollection.Add(repo), IsNil)
 
 	changesFiles, failedFiles := CollectChangesFiles(
 		[]string{"testdata/dbgsym-with-source-version"}, s.Reporter)
@@ -145,8 +142,7 @@ func (s *ChangesSuite) TestImportDbgsymWithVersionedSourceField(c *C) {
 
 	_, failedFiles, err := ImportChangesFiles(
 		changesFiles, s.Reporter, true, true, false, true, &NullVerifier{},
-		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.reflistCollection, s.packagePool,
+		template.Must(template.New("test").Parse("test")), s.progress, s.localRepoCollection, s.packageCollection, s.packagePool, func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
-		func(database.ReaderWriter) aptly.ChecksumStorage { return s.checksumStorage },
 		nil, nil)
 	c.Assert(err, IsNil)
 	c.Check(failedFiles, IsNil)

deb/checksum_collection_test.go

@@ -28,7 +28,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
 }
 
 func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
-	s.db.Close()
+	_ = s.db.Close()
 }
 
 func (s *ChecksumCollectionSuite) TestFlow(c *C) {

deb/collections.go

@@ -16,7 +16,6 @@ type CollectionFactory struct {
 	snapshots      *SnapshotCollection
 	localRepos     *LocalRepoCollection
 	publishedRepos *PublishedRepoCollection
-	reflists       *RefListCollection
 	checksums      *ChecksumCollection
 }
 
@@ -92,17 +91,6 @@ func (factory *CollectionFactory) PublishedRepoCollection() *PublishedRepoCollec
 	return factory.publishedRepos
 }
 
-func (factory *CollectionFactory) RefListCollection() *RefListCollection {
-	factory.Lock()
-	defer factory.Unlock()
-
-	if factory.reflists == nil {
-		factory.reflists = NewRefListCollection(factory.db)
-	}
-
-	return factory.reflists
-}
-
 // ChecksumCollection returns (or creates) new ChecksumCollection
 func (factory *CollectionFactory) ChecksumCollection(db database.ReaderWriter) aptly.ChecksumStorage {
 	factory.Lock()

deb/deb.go

@@ -17,7 +17,7 @@ import (
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/kjk/lzma"
 	"github.com/klauspost/compress/zstd"
-	"github.com/smira/go-xz"
+	xz "github.com/smira/go-xz"
 )
 
 // Source kinds
@@ -35,7 +35,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
+	defer func() { _ = file.Close() }()
 
 	library := ar.NewReader(file)
 	for {
@@ -66,14 +66,14 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
 				}
-				defer ungzip.Close()
+				defer func() { _ = ungzip.Close() }()
 				tarInput = ungzip
 			case "control.tar.xz":
 				unxz, err := xz.NewReader(bufReader)
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
 				}
-				defer unxz.Close()
+				defer func() { _ = unxz.Close() }()
 				tarInput = unxz
 			case "control.tar.zst":
 				unzstd, err := zstd.NewReader(bufReader)
@@ -116,10 +116,10 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 	if err != nil {
 		return nil, err
 	}
-	defer file.Close()
+	defer func() { _ = file.Close() }()
 
 	isClearSigned, err := verifier.IsClearSigned(file)
-	file.Seek(0, 0)
+	_, _ = file.Seek(0, 0)
 
 	if err != nil {
 		return nil, err
@@ -132,7 +132,7 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
 		if err != nil {
 			return nil, err
 		}
-		defer text.Close()
+		defer func() { _ = text.Close() }()
 	} else {
 		text = file
 	}
@@ -181,7 +181,7 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 					if err != nil {
 						return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
 					}
-					defer ungzip.Close()
+					defer func() { _ = ungzip.Close() }()
 					tarInput = ungzip
 				}
 			case "data.tar.bz2":
@@ -191,11 +191,11 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
 				if err != nil {
 					return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
 				}
-				defer unxz.Close()
+				defer func() { _ = unxz.Close() }()
 				tarInput = unxz
 			case "data.tar.lzma":
 				unlzma := lzma.NewReader(bufReader)
-				defer unlzma.Close()
+				defer func() { _ = unlzma.Close() }()
 				tarInput = unlzma
 			case "data.tar.zst":
 				unzstd, err := zstd.NewReader(bufReader)

deb/find_dangling.go

@@ -0,0 +1,45 @@
+package deb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/aptly-dev/aptly/database"
+)
+
+// FindDanglingReferences finds references that exist in the given PackageRefList, but not in the given PackageCollection.
+// It returns all such references, so they can be removed from the database.
+func FindDanglingReferences(reflist *PackageRefList, packages *PackageCollection) (dangling *PackageRefList, err error) {
+	dangling = &PackageRefList{}
+
+	err = reflist.ForEach(func(key []byte) error {
+		ok, err := isDangling(packages, key)
+		if err != nil {
+			return err
+		}
+
+		if ok {
+			dangling.Refs = append(dangling.Refs, key)
+		}
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return dangling, nil
+}
+
+func isDangling(packages *PackageCollection, key []byte) (bool, error) {
+	_, err := packages.ByKey(key)
+	if errors.Is(err, database.ErrNotFound) {
+		return true, nil
+	}
+
+	if err != nil {
+		return false, fmt.Errorf("get reference %q: %w", key, err)
+	}
+
+	return false, nil
+}

deb/find_dangling_test.go

@@ -0,0 +1,46 @@
+package deb_test
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/aptly-dev/aptly/database/goleveldb"
+	"github.com/aptly-dev/aptly/deb"
+)
+
+func TestFindDanglingReferences(t *testing.T) {
+	reflist := deb.NewPackageRefList()
+	reflist.Refs = [][]byte{[]byte("P existing 1.2.3"), []byte("P dangling 1.2.3")}
+
+	db, _ := goleveldb.NewOpenDB(t.TempDir())
+	packages := deb.NewPackageCollection(db)
+
+	if err := packages.Update(&deb.Package{Name: "existing", Version: "1.2.3"}); err != nil {
+		t.Fatal(err)
+	}
+
+	dangling, err := deb.FindDanglingReferences(reflist, packages)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exp := &deb.PackageRefList{
+		Refs: [][]byte{[]byte("P dangling 1.2.3")},
+	}
+
+	compareRefs(t, exp, dangling)
+}
+
+func compareRefs(t *testing.T, exp, got *deb.PackageRefList) {
+	t.Helper()
+
+	if len(exp.Refs) != len(got.Refs) {
+		t.Fatalf("refs length mismatch: exp %d, got %d", len(exp.Refs), len(got.Refs))
+	}
+
+	for i := range exp.Refs {
+		if !bytes.Equal(exp.Refs[i], got.Refs[i]) {
+			t.Fatalf("refs do not match: exp %q, got %q", exp.Refs[i], got.Refs[i])
+		}
+	}
+}

deb/format.go

@@ -26,8 +26,11 @@ var (
 		"Version",
 		"Codename",
 		"Date",
+		"Valid-Until",
 		"NotAutomatic",
 		"ButAutomaticUpgrades",
+		"Acquire-By-Hash",
+		"Signed-By",
 		"Architectures",
 		"Architecture",
 		"Components",

deb/format_test.go

@@ -163,7 +163,7 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
 func (s *ControlFileSuite) TestLongFields(c *C) {
 	f, err := os.Open("long.stanza")
 	c.Assert(err, IsNil)
-	defer f.Close()
+	defer func() { _ = f.Close() }()
 
 	r := NewControlFileReader(f, false, false)
 	stanza, e := r.ReadStanza()

deb/graph.go

@@ -12,15 +12,15 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	var err error
 
 	graph := gographviz.NewEscape()
-	graph.SetDir(true)
+	_ = graph.SetDir(true)
-	graph.SetName("aptly")
+	_ = graph.SetName("aptly")
 
 	var labelStart string
 	var labelEnd string
 
 	switch layout {
 	case "vertical":
-		graph.AddAttr("aptly", "rankdir", "LR")
+		_ = graph.AddAttr("aptly", "rankdir", "LR")
 		labelStart = ""
 		labelEnd = ""
 	case "horizontal":
@@ -33,12 +33,12 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	existingNodes := map[string]bool{}
 
 	err = collectionFactory.RemoteRepoCollection().ForEach(func(repo *RemoteRepo) error {
-		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.RemoteRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
 
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkgoldenrod1",
@@ -55,12 +55,12 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 	}
 
 	err = collectionFactory.LocalRepoCollection().ForEach(func(repo *LocalRepo) error {
-		e := collectionFactory.LocalRepoCollection().LoadComplete(repo, collectionFactory.RefListCollection())
+		e := collectionFactory.LocalRepoCollection().LoadComplete(repo)
 		if e != nil {
 			return e
 		}
 
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "mediumseagreen",
@@ -75,13 +75,13 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		return nil, err
 	}
 
-	collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
+	_ = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
 		existingNodes[snapshot.UUID] = true
 		return nil
 	})
 
 	err = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
-		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot, collectionFactory.RefListCollection())
+		e := collectionFactory.SnapshotCollection().LoadComplete(snapshot)
 		if e != nil {
 			return e
 		}
@@ -91,7 +91,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			description = "Snapshot from repo"
 		}
 
-		graph.AddNode("aptly", snapshot.UUID, map[string]string{
+		_ = graph.AddNode("aptly", snapshot.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "cadetblue1",
@@ -103,7 +103,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 			for _, uuid := range snapshot.SourceIDs {
 				_, exists := existingNodes[uuid]
 				if exists {
-					graph.AddEdge(uuid, snapshot.UUID, true, nil)
+					_ = graph.AddEdge(uuid, snapshot.UUID, true, nil)
 				}
 			}
 		}
@@ -114,8 +114,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		return nil, err
 	}
 
-	collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
+	_ = collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
-		graph.AddNode("aptly", repo.UUID, map[string]string{
+		_ = graph.AddNode("aptly", repo.UUID, map[string]string{
 			"shape":     "Mrecord",
 			"style":     "filled",
 			"fillcolor": "darkolivegreen1",
@@ -127,7 +127,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
 		for _, uuid := range repo.Sources {
 			_, exists := existingNodes[uuid]
 			if exists {
-				graph.AddEdge(uuid, repo.UUID, true, nil)
+				_ = graph.AddEdge(uuid, repo.UUID, true, nil)
 			}
 		}
 

deb/index_files.go

@@ -59,24 +59,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
 		if file.discardable {
 			return nil
 		}
-		file.BufWriter()
+		_, _ = file.BufWriter()
 	}
 
 	err := file.w.Flush()
 	if err != nil {
-		file.tempFile.Close()
+		_ = file.tempFile.Close()
 		return fmt.Errorf("unable to write to index file: %s", err)
 	}
 
 	if file.compressable {
 		err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
 		if err != nil {
-			file.tempFile.Close()
+			_ = file.tempFile.Close()
 			return fmt.Errorf("unable to compress index file: %s", err)
 		}
 	}
 
-	file.tempFile.Close()
+	_ = file.tempFile.Close()
 
 	exts := []string{""}
 	cksumExts := exts
@@ -220,11 +220,11 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
 				// If we managed to resolve the link target: delete it. This is the
 				// oldest physical index file we no longer need. Once we drop our
 				// old symlink we'll essentially forget about it existing at all.
-				file.parent.publishedStorage.Remove(linkTarget)
+				_ = file.parent.publishedStorage.Remove(linkTarget)
 			}
-			file.parent.publishedStorage.Remove(oldIndexPath)
+			_ = file.parent.publishedStorage.Remove(oldIndexPath)
 		}
-		file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
+		_ = file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
 	}
 
 	// create symlink