[not4debian] ci: run unit tests in docker

- run separate unit-test job
- build docker
- allow make docker-unit-tests in ci

.gitignore

@@ -38,6 +38,7 @@ man/aptly.1.ronn
 system/env/
 
 # created by make build for release artifacts
+VERSION
 aptly.test
 
 build/
@@ -73,42 +74,3 @@ docs/docs.go
 docs/swagger.json
 docs/swagger.yaml
 docs/swagger.conf
-.secrets
-
-# Coverage reports
-*.out
-coverage.html
-*_coverage.html
-
-# Binaries
-aptly-binary
-aptly-test
-
-# Downloaded archives
-*.tar.gz
-
-# Test artifacts
-test_results.log
-
-# Python virtual environments
-system/venv/
-venv/
-
-# act local CI runner
-.actrc
-
-# Backup files
-*.backup
-*.bak
-
-# Temporary directories
-coverage/
-scripts/
-
-# Binary executables
-aptly/aptly
-
-# Coverage reports
-coverage_report.html
-*.coverage
-coverage.out

.golangci.yml

@@ -1,211 +1,11 @@
-# golangci-lint configuration for aptly
-# Run with: golangci-lint run
-
-run:
-  # Timeout for analysis
-  timeout: 5m
-  
-  # Include test files
-  tests: true
-
-output:
-  # Format of output
-  formats:
-    - format: colored-line-number
-  
-  # Print lines of code with issue
-  print-issued-lines: true
-  
-  # Print linter name in the end of issue text
-  print-linter-name: true
-
+version: "2"
 linters:
-  enable:
-    # Default linters
-    - errcheck      # Check for unchecked errors
-    - gosimple      # Simplify code
-    - govet         # Go vet
-    - ineffassign   # Detect ineffectual assignments
-    - staticcheck   # Static analysis
-    - typecheck     # Type checking
-    - unused        # Find unused code
-    
-    # Additional linters for code quality
-    - bodyclose     # Check HTTP response body is closed
-    - dupl          # Code duplication
-    - copyloopvar   # Check loop variable export (replacement for exportloopref)
-    - gocognit      # Cognitive complexity
-    - gocritic      # Opinionated linter
-    - gocyclo       # Cyclomatic complexity
-    - gofmt         # Formatting
-    - goimports     # Import formatting
-    - revive        # Fast, configurable linter
-    - unconvert     # Unnecessary type conversions
-    - unparam       # Unused function parameters
-    - gosec         # Security issues
-    - prealloc      # Preallocate slices
-    - predeclared   # Shadowing of predeclared identifiers
-    - makezero      # Make slice with non-zero length
-    - nakedret      # Naked returns in long functions
-    
-  disable:
-    # Disabled because they're too strict or noisy
-    - exhaustive    # Too strict for switch statements
-    - wsl           # Whitespace linter (too opinionated)
-    - godox         # TODO/FIXME comments
-    - gochecknoglobals # We use some globals
-    - gochecknoinits   # We use init functions
-
-linters-settings:
-  # errcheck
-  errcheck:
-    # Report about not checking of errors in type assertions
-    check-type-assertions: true
-    # Report about assignment of errors to blank identifier
-    check-blank: true
-    # Exclude some functions from checking
-    exclude-functions:
-      - io/ioutil.ReadFile
-      - io.Copy(*bytes.Buffer)
-      - io.Copy(os.Stdout)
-
-  # govet
-  govet:
-    enable-all: true
-    disable:
-      - fieldalignment # Too many false positives
-
-  # gocyclo
-  gocyclo:
-    min-complexity: 15
-
-  # gocognit
-  gocognit:
-    min-complexity: 20
-
-  # dupl
-  dupl:
-    threshold: 200
-
-  # gocritic
-  gocritic:
-    enabled-tags:
-      - diagnostic
-      - performance
-      - style
-    disabled-checks:
-      - commentedOutCode
-      - whyNoLint
-
-  # gosec
-  gosec:
-    severity: low
-    confidence: low
-    excludes:
-      - G404 # Weak random for non-crypto use is ok
-
-  # revive
-  revive:
-    rules:
-      - name: blank-imports
-      - name: context-as-argument
-      - name: context-keys-type
-      - name: dot-imports
-      - name: empty-block
-      - name: error-naming
-      - name: error-return
-      - name: error-strings
-      - name: errorf
-      - name: exported
-      - name: increment-decrement
-      - name: indent-error-flow
-      - name: package-comments
-      - name: range
-      - name: receiver-naming
-      - name: time-naming
-      - name: unexported-return
-      - name: var-declaration
-      - name: var-naming
-
-  # goimports
-  goimports:
-    local-prefixes: github.com/aptly-dev/aptly
-
-  # gofmt
-  gofmt:
-    simplify: true
-
-  # unparam
-  unparam:
-    check-exported: false
-
-  # nakedret
-  nakedret:
-    max-func-lines: 30
-
-  # prealloc
-  prealloc:
-    simple: true
-    range-loops: true
-    for-loops: false
-
-issues:
-  # Maximum issues count per one linter
-  max-issues-per-linter: 0
-  
-  # Maximum count of issues with the same text
-  max-same-issues: 0
-  
-  # Skip directories
-  exclude-dirs:
-    - vendor
-    - testdata
-    - system/files
-    
-  # Skip files matching these patterns
-  exclude-files:
-    - ".*\\.pb\\.go$"
-    - ".*\\.gen\\.go$"
-  
-  # Exclude some linters from running on tests files
-  exclude-rules:
-    - path: _test\.go
-      linters:
-        - dupl
-        - gosec
-        - gocognit
-        - gocyclo
-    
-    # Exclude some linters from running on generated files
-    - path: ".*\\.gen\\.go$"
-      linters:
-        - all
-    
-    # Exclude known issues in vendor
-    - path: vendor/
-      linters:
-        - all
-    
-    # Allow fmt.Printf in main/cmd
-    - path: (cmd|main)\.go
-      linters:
-        - forbidigo
-
-  # Independently from option `exclude` we use default exclude patterns
-  exclude-use-default: true
-  
-  # Fix found issues (if it's supported by the linter)
-  fix: false
-
-severity:
-  # Set the default severity for issues
-  default-severity: warning
-  
-  # The list of ids of default excludes to include or disable
-  rules:
-    - linters:
-        - gosec
-      severity: info
-    - linters:
-        - dupl
-      severity: info
+  settings:
+    staticcheck:
+      checks:
+      - "all"
+      - "-QF1004"  # could use strings.ReplaceAll instead
+      - "-QF1012"  # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))
+      - "-QF1003"  # could use tagged switch
+      - "-ST1000"  # at least one file in a package should have a package comment
+      - "-QF1001"  # could apply De Morgan's law

AUTHORS

@@ -69,3 +69,10 @@ List of contributors, in chronological order:
 * Leigh London (https://github.com/leighlondon)
 * Gordian Schoenherr (https://github.com/schoenherrg)
 * Silke Hofstra (https://github.com/silkeh)
+* Itay Porezky (https://github.com/itayporezky)
+* JupiterRider (https://github.com/JupiterRider)
+* Tobias Assarsson (https://github.com/daedaluz)
+* Yaksh Bariya (https://github.com/thunder-coding)
+* Brian Witt (https://github.com/bwitt)
+* Ales Bregar (https://github.com/abregar)
+* Tim Foerster (https://github.com/tonobo)

CONTRIBUTING.md

@@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu
 ### List of Repositories
 
 * [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
-* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
+* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
 * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
   fixtures for aptly functional tests
 
@@ -158,7 +158,7 @@ This section describes local setup to start contributing to aptly.
 
 #### Dependencies
 
-Building aptly requires go version 1.24.
+Building aptly requires go version 1.22.
 
 On Debian bookworm with backports enabled, go can be installed with:
 
@@ -178,149 +178,6 @@ To install aptly into `$GOPATH/bin`, run:
 
     make install
 
-#### Platform-Specific Setup
-
-##### macOS
-
-This guide explains how to run aptly tests on macOS, including Apple Silicon (M1/M2) machines.
-
-###### Prerequisites
-
-1. **Install Go** (1.24 or later):
-   ```bash
-   brew install go
-   ```
-
-2. **Install Docker** (for etcd and other services):
-   ```bash
-   brew install --cask docker
-   ```
-
-3. **Install test dependencies**:
-   ```bash
-   # Add Go binaries to PATH
-   export PATH=$PATH:~/go/bin
-   
-   # Install swag for API documentation
-   go install github.com/swaggo/swag/cmd/swag@latest
-   
-   # Install other tools
-   brew install etcd  # Optional: for local etcd instead of Docker
-   ```
-
-###### Running Tests on macOS
-
-**Option 1: Using Docker Compose (Recommended)**
-
-```bash
-# Start test services
-docker-compose -f docker-compose.ci.yml up -d etcd
-
-# Run tests
-PATH=$PATH:~/go/bin make test
-```
-
-**Option 2: Using Local etcd**
-
-```bash
-# Install and start etcd locally
-brew services start etcd
-
-# Run tests with local etcd
-ETCD_ENDPOINTS=localhost:2379 go test ./...
-```
-
-**Option 3: Run Specific Test Suites**
-
-```bash
-# Fix VERSION file if needed
-echo "1.5.0" > VERSION
-
-# Run unit tests only
-PATH=$PATH:~/go/bin make test-unit GOTEST="go test -short -timeout=5m"
-
-# Run specific packages
-go test ./deb ./s3 ./utils ./context -short -v
-
-# Run with race detection
-go test -race ./deb ./s3 ./utils -short
-```
-
-###### macOS-Specific Considerations
-
-1. **CPU Architecture**: The install scripts now support both Intel (x86_64) and Apple Silicon (arm64).
-
-2. **File System**: macOS is case-insensitive by default, which may affect some tests.
-
-3. **Network**: Some tests may require adjusting firewall settings.
-
-4. **Timeouts**: Some tests may need longer timeouts on macOS:
-   ```bash
-   go test -timeout=10m ./...
-   ```
-
-###### Troubleshooting on macOS
-
-**etcd Installation Fails**
-
-If the automatic etcd installation fails, use Docker or Homebrew:
-```bash
-# Using Docker
-docker run -d -p 2379:2379 --name etcd quay.io/coreos/etcd:latest
-
-# Using Homebrew
-brew install etcd
-etcd --listen-client-urls http://0.0.0.0:2379 --advertise-client-urls http://localhost:2379
-```
-
-**Test Timeouts**
-
-Increase timeouts for slower tests:
-```bash
-go test -timeout=30m ./...
-```
-
-**Race Detector Issues**
-
-The race detector may be slower on macOS. Disable for faster runs:
-```bash
-go test ./... -short
-```
-
-###### CI Integration for macOS
-
-For GitHub Actions on macOS:
-
-```yaml
-jobs:
-  test-macos:
-    runs-on: macos-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: '1.24'
-      
-      - name: Install dependencies
-        run: |
-          brew install etcd
-          go install github.com/swaggo/swag/cmd/swag@latest
-      
-      - name: Run tests
-        run: |
-          export PATH=$PATH:~/go/bin
-          make test
-```
-
-###### Test Coverage on macOS
-
-Generate coverage reports:
-```bash
-go test -coverprofile=coverage.out ./...
-go tool cover -html=coverage.out -o coverage.html
-open coverage.html
-```
-
 #### Unit-tests
 
 aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
@@ -377,59 +234,6 @@ There are some packages available under `system/files/` directory which are used
 this default location. You can run aptly under different user or by using non-default config location with non-default
 aptly root directory.
 
-### Continuous Integration (CI)
-
-aptly uses GitHub Actions for continuous integration. The CI pipeline includes:
-
-- **Quick checks**: Code formatting, go vet, mod tidy, and flake8 linting
-- **Security scanning**: govulncheck and Trivy vulnerability scanning  
-- **Linting**: golangci-lint with extensive checks
-- **Unit tests**: With race detection on Go 1.23 and 1.24
-- **Integration tests**: Full system tests with cloud storage backends
-- **Benchmarks**: Performance testing
-- **Extended tests**: Combined unit tests and benchmarks with coverage merging
-- **Cross-platform builds**: Binaries for Linux, macOS, Windows, FreeBSD (multiple architectures)
-- **Debian packages**: Built for Debian (buster, bullseye, bookworm, trixie) and Ubuntu (focal, jammy, noble)
-- **Docker images**: Multi-architecture container images (linux/amd64, linux/arm64)
-
-All pull requests must pass CI checks before merging. Build artifacts are available for download from GitHub Actions runs with the following retention:
-- CI builds: 7 days
-- Tagged releases: 90 days
-
-#### Testing CI Locally with act
-
-You can test GitHub Actions workflows locally using [act](https://github.com/nektos/act):
-
-```bash
-# Install act
-brew install act              # macOS
-# or
-curl https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash  # Linux
-
-# Run default push event
-act
-
-# Run pull request event
-act pull_request
-
-# Run specific job
-act -j test-unit
-
-# Run with specific matrix values
-act -j test-unit --matrix go:1.24
-
-# List all available jobs
-act -l
-```
-
-For Apple Silicon Macs, use: `act --container-architecture linux/amd64`
-
-Common use cases:
-- Test a job before pushing: `act -j quick-checks`
-- Test PR workflows: Create a PR event file and run `act pull_request -e pr-event.json`
-- Debug failures: `act -j failing-job -v` for verbose output
-- Use secrets: Create `.secrets` file with `KEY=value` format and run `act --secret-file .secrets`
-
 ### man Page
 
 aptly is using combination of [Go templates](http://godoc.org/text/template) and automatically generated text to build `aptly.1` man page. If either source

Makefile

@@ -1,74 +1,45 @@
-# Modern Makefile for aptly with improved tooling and practices
+GOPATH=$(shell go env GOPATH)
+VERSION=$(shell make -s version)
+PYTHON?=python3
+BINPATH?=$(GOPATH)/bin
+GOLANGCI_LINT_VERSION=v2.0.2  # version supporting go 1.24
+COVERAGE_DIR?=$(shell mktemp -d)
+GOOS=$(shell go env GOHOSTOS)
+GOARCH=$(shell go env GOHOSTARCH)
 
-SHELL := /bin/bash
-.DEFAULT_GOAL := help
-.PHONY: help
+export PODMAN_USERNS = keep-id
+DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
 
-# Version and build info
-BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%SZ")
-GIT_COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
+# Setting TZ for certificates
+export TZ=UTC
+# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
+export TEST_FAKETIME := 2025-01-02 03:04:05
 
-# Go parameters
-GOCMD := go
-GOBUILD := $(GOCMD) build
-GOTEST := $(GOCMD) test
-GOGET := $(GOCMD) get
-GOMOD := $(GOCMD) mod
-GOFMT := gofmt
-GOPATH := $(shell go env GOPATH)
-BINPATH := $(GOPATH)/bin
-GOOS := $(shell go env GOHOSTOS)
-GOARCH := $(shell go env GOHOSTARCH)
-
-# OS detection
-UNAME_S := $(shell uname -s)
-ifeq ($(UNAME_S),Darwin)
-    OS_TYPE := macos
+# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
+ifeq ($(COVERAGE_SKIP),1)
+COVERAGE_ARG_BUILD :=
+COVERAGE_ARG_TEST := --coverage-skip
 else
-    OS_TYPE := linux
+COVERAGE_ARG_BUILD := -coverpkg="./..."
+COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
 endif
 
-# Tool versions
-GOLANGCI_VERSION := v1.64.5
-AIR_VERSION := v1.52.3
-SWAG_VERSION := v1.16.4
-GOVULNCHECK_VERSION := latest
+# export CAPUTRE=1 for regenrating test gold files
+ifeq ($(CAPTURE),1)
+CAPTURE_ARG := --capture
+endif
 
-# Build parameters
-BINARY_NAME := aptly
-BUILD_DIR := build
-COVERAGE_DIR := coverage
-COVERAGE_FILE := $(COVERAGE_DIR)/coverage.out
+help:  ## Print this help
+	@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-# Docker parameters
-DOCKER_IMAGE := aptly/aptly
-DOCKER_TAG := $(VERSION)
+prepare:  ## Install go module dependencies
+	# Prepare go modules
+	go mod verify
+	go mod tidy -v
+	# Generate VERSION file
+	go generate
 
-# Colors for output
-COLOR_RESET := \033[0m
-COLOR_BOLD := \033[1m
-COLOR_GREEN := \033[32m
-COLOR_YELLOW := \033[33m
-COLOR_RED := \033[31m
-COLOR_BLUE := \033[34m
-
-##@ General
-
-help: ## Display this help
-	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
-
-version: ## Show version
-	@ci="" ; \
-	if [ "`make -s releasetype`" = "ci" ]; then \
-		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
-	fi ; \
-	if which dpkg-parsechangelog > /dev/null 2>&1; then \
-		echo `dpkg-parsechangelog -S Version`$$ci; \
-	else \
-		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
-	fi
-
-releasetype: # Print release type: ci (on any branch/commit), release (on a tag)
+releasetype:  # Print release type: ci (on any branch/commit), release (on a tag)
 	@reltype=ci ; \
 	gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
 	if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
@@ -79,236 +50,195 @@ releasetype: # Print release type: ci (on any branch/commit), release (on a tag)
 	fi ; \
 	echo $$reltype
 
-##@ Development
-
-prepare: ## Prepare development environment
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Preparing development environment...$(COLOR_RESET)"
-	$(GOMOD) download
-	$(GOMOD) verify
-	$(GOMOD) tidy -v
-	@go generate ./...
-
-dev-tools: ## Install development tools
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Installing development tools...$(COLOR_RESET)"
-	@go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_VERSION)
-	@go install github.com/air-verse/air@$(AIR_VERSION)
-	@go install github.com/swaggo/swag/cmd/swag@$(SWAG_VERSION)
-	@go install golang.org/x/vuln/cmd/govulncheck@$(GOVULNCHECK_VERSION)
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Development tools installed$(COLOR_RESET)"
-
-##@ Build
-
-build: prepare swagger ## Build aptly binary
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Building aptly...$(COLOR_RESET)"
-	@mkdir -p $(BUILD_DIR)
-	$(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME) .
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Build complete: $(BUILD_DIR)/$(BINARY_NAME)$(COLOR_RESET)"
-
-build-all: prepare swagger ## Build for all platforms
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Building for all platforms...$(COLOR_RESET)"
-	@mkdir -p $(BUILD_DIR)
-	# Linux
-	GOOS=linux GOARCH=amd64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64
-	GOOS=linux GOARCH=arm64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64
-	# macOS
-	GOOS=darwin GOARCH=amd64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-amd64
-	GOOS=darwin GOARCH=arm64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64
-	# Windows
-	GOOS=windows GOARCH=amd64 $(GOBUILD) -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Multi-platform build complete$(COLOR_RESET)"
-
-install: build ## Install aptly to GOPATH/bin
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Installing aptly...$(COLOR_RESET)"
-	@cp $(BUILD_DIR)/$(BINARY_NAME) $(BINPATH)/
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Installed to $(BINPATH)/$(BINARY_NAME)$(COLOR_RESET)"
-
-##@ Testing
-
-test: prepare test-unit test-integration ## Run all tests
-
-test-unit: prepare swagger etcd-install ## Run unit tests
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running unit tests...$(COLOR_RESET)"
-	@mkdir -p $(COVERAGE_DIR)
-	$(GOTEST) -v -race -coverprofile=$(COVERAGE_DIR)/unit.out -covermode=atomic ./...
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Unit tests complete$(COLOR_RESET)"
-
-test-integration: prepare swagger etcd-install ## Run integration tests
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running integration tests...$(COLOR_RESET)"
-	@mkdir -p $(COVERAGE_DIR)
-	# Download fixtures if needed
-	@if [ ! -e ~/aptly-fixture-db ]; then \
-		git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; \
+version:  ## Print aptly version
+	@ci="" ; \
+	if [ "`make -s releasetype`" = "ci" ]; then \
+		ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
+	fi ; \
+	if which dpkg-parsechangelog > /dev/null 2>&1; then \
+		echo `dpkg-parsechangelog -S Version`$$ci; \
+	else \
+		echo `grep ^aptly -m1  debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
 	fi
-	@if [ ! -e ~/aptly-fixture-pool ]; then \
-		git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; \
-	fi
-	# Run system tests
-	PATH=$(BINPATH):$$PATH python3 system/run.py --coverage-dir $(COVERAGE_DIR) $(TEST)
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Integration tests complete$(COLOR_RESET)"
 
-test-race: ## Run tests with race detector
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running tests with race detector...$(COLOR_RESET)"
-	$(GOTEST) -race -short ./...
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Race detection complete$(COLOR_RESET)"
+swagger-install:
+	# Install swag
+	@test -f $(BINPATH)/swag || GOOS= GOARCH= go install github.com/swaggo/swag/cmd/swag@latest
+	# Generate swagger.conf
+	cp docs/swagger.conf.tpl docs/swagger.conf
+	echo "// @version $(VERSION)" >> docs/swagger.conf
 
-coverage: test ## Generate coverage report
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Generating coverage report...$(COLOR_RESET)"
-	@mkdir -p $(COVERAGE_DIR)
-	@go tool cover -html=$(COVERAGE_DIR)/unit.out -o $(COVERAGE_DIR)/coverage.html
-	@go tool cover -func=$(COVERAGE_DIR)/unit.out
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Coverage report: $(COVERAGE_DIR)/coverage.html$(COLOR_RESET)"
-
-benchmark: ## Run benchmarks
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running benchmarks...$(COLOR_RESET)"
-	$(GOTEST) -bench=. -benchmem ./deb ./files ./utils
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Benchmarks complete$(COLOR_RESET)"
-
-##@ Code Quality
-
-lint: dev-tools ## Run linters
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running linters...$(COLOR_RESET)"
-	@golangci-lint run --timeout=5m
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Linting complete$(COLOR_RESET)"
-
-fmt: ## Format code
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Formatting code...$(COLOR_RESET)"
-	@$(GOFMT) -w -s .
-	@$(GOMOD) tidy
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Code formatted$(COLOR_RESET)"
-
-vet: ## Run go vet
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running go vet...$(COLOR_RESET)"
-	@go vet ./...
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Vet complete$(COLOR_RESET)"
-
-security: dev-tools ## Run security checks
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Running security checks...$(COLOR_RESET)"
-	@govulncheck ./...
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Security check complete$(COLOR_RESET)"
-
-##@ Dependencies
-
-deps-update: ## Update dependencies
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Updating dependencies...$(COLOR_RESET)"
-	@./scripts/update-deps.sh
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Dependencies updated$(COLOR_RESET)"
-
-deps-check: ## Check for outdated dependencies
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Checking for outdated dependencies...$(COLOR_RESET)"
-	@go list -u -m all | grep '\[' || echo "All dependencies are up to date!"
-
-deps-graph: ## Generate dependency graph
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Generating dependency graph...$(COLOR_RESET)"
-	@go mod graph | grep -v '@' | sort | uniq
-
-##@ Documentation
-
-swagger: swagger-install ## Generate Swagger documentation
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Generating Swagger documentation...$(COLOR_RESET)"
-	@cp docs/swagger.conf.tpl docs/swagger.conf
-	@echo "// @version $(VERSION)" >> docs/swagger.conf
-	@swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Swagger docs generated$(COLOR_RESET)"
-
-swagger-install: ## Install swagger tools
-	@test -f $(BINPATH)/swag || go install github.com/swaggo/swag/cmd/swag@$(SWAG_VERSION)
-
-docs: swagger ## Generate all documentation
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Documentation generated$(COLOR_RESET)"
-
-##@ Development Server
-
-serve: dev-tools prepare ## Run development server with hot reload
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Starting development server...$(COLOR_RESET)"
-	@cp debian/aptly.conf ~/.aptly.conf || true
-	@sed -i.bak '/enable_swagger_endpoint/s/false/true/' ~/.aptly.conf || true
-	@air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' \
-		-build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu \
-		-- api serve -listen 0.0.0.0:3142
-
-##@ Docker
-
-docker-build: ## Build Docker image
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Building Docker image...$(COLOR_RESET)"
-	docker build -t $(DOCKER_IMAGE):$(DOCKER_TAG) -t $(DOCKER_IMAGE):latest .
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Docker image built: $(DOCKER_IMAGE):$(DOCKER_TAG)$(COLOR_RESET)"
-
-docker-push: ## Push Docker image
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Pushing Docker image...$(COLOR_RESET)"
-	docker push $(DOCKER_IMAGE):$(DOCKER_TAG)
-	docker push $(DOCKER_IMAGE):latest
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Docker image pushed$(COLOR_RESET)"
-
-##@ Cleanup
-
-clean: ## Clean build artifacts
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Cleaning build artifacts...$(COLOR_RESET)"
-	@rm -rf $(BUILD_DIR) $(COVERAGE_DIR)
-	@rm -f docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
-	@rm -rf obj-* *.out *.test
-	@docker-compose -f docker-compose.ci.yml down || true
-	@docker volume prune -f || true
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Clean complete$(COLOR_RESET)"
-
-clean-deps: ## Clean dependency cache
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Cleaning dependency cache...$(COLOR_RESET)"
-	@go clean -modcache
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Dependency cache cleaned$(COLOR_RESET)"
-
-##@ CI/CD
-
-ci: prepare lint test security ## Run CI pipeline
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ CI pipeline complete$(COLOR_RESET)"
-
-release: clean build-all ## Prepare release artifacts
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Preparing release...$(COLOR_RESET)"
-	@mkdir -p $(BUILD_DIR)/release
-	@for file in $(BUILD_DIR)/$(BINARY_NAME)-*; do \
-		base=$$(basename $$file); \
-		tar -czf $(BUILD_DIR)/release/$$base.tar.gz -C $(BUILD_DIR) $$base; \
-	done
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Release artifacts ready in $(BUILD_DIR)/release$(COLOR_RESET)"
-
-##@ Utilities
-
-etcd-install: ## Install etcd for testing
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Starting fresh etcd container...$(COLOR_RESET)"
-	@docker-compose -f docker-compose.ci.yml down etcd 2>/dev/null || true
-	@docker-compose -f docker-compose.ci.yml rm -f -v etcd 2>/dev/null || true
-	@docker volume rm aptly_etcd-data 2>/dev/null || true
-	@docker-compose -f docker-compose.ci.yml up -d etcd
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Waiting for etcd to be ready...$(COLOR_RESET)"
-	@sleep 5
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ etcd started in Docker$(COLOR_RESET)"
-
-etcd-start: ## Start etcd
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Starting fresh etcd container...$(COLOR_RESET)"
-	@docker-compose -f docker-compose.ci.yml down etcd 2>/dev/null || true
-	@docker-compose -f docker-compose.ci.yml rm -f -v etcd 2>/dev/null || true
-	@docker volume rm aptly_etcd-data 2>/dev/null || true
-	@docker-compose -f docker-compose.ci.yml up -d etcd
-	@sleep 5
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ etcd started in Docker$(COLOR_RESET)"
-
-etcd-stop: ## Stop etcd
-	@docker-compose -f docker-compose.ci.yml down etcd 2>/dev/null || true
-	@docker-compose -f docker-compose.ci.yml rm -f -v etcd 2>/dev/null || true
-	@docker volume rm aptly_etcd-data 2>/dev/null || true
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ etcd stopped and cleaned$(COLOR_RESET)"
-
-azurite-start: ## Start Azurite (Azure Storage Emulator) for tests
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Starting Azurite...$(COLOR_RESET)"
-	@azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
+azurite-start:
+	azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
 	echo $$! > ~/.azurite.pid
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Azurite started (PID: $$(cat ~/.azurite.pid))$(COLOR_RESET)"
 
-azurite-stop: ## Stop Azurite
-	@echo -e "$(COLOR_YELLOW)$(COLOR_BOLD)Stopping Azurite...$(COLOR_RESET)"
-	@-kill `cat ~/.azurite.pid` 2>/dev/null || true
-	@rm -f ~/.azurite.pid
-	@echo -e "$(COLOR_GREEN)$(COLOR_BOLD)✓ Azurite stopped$(COLOR_RESET)"
+azurite-stop:
+	@kill `cat ~/.azurite.pid`
 
-.PHONY: all build build-all install test test-unit test-integration test-race coverage benchmark \
-	lint fmt vet security deps-update deps-check deps-graph docs swagger swagger-install serve \
-	docker-build docker-push clean clean-deps ci release prepare dev-tools etcd-install etcd-start etcd-stop \
-	azurite-start azurite-stop
+swagger: #swagger-install
+	# Generate swagger docs
+	#@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
+
+etcd-install:
+	# Install etcd
+	test -d /tmp/aptly-etcd || system/t13_etcd/install-etcd.sh
+
+flake8:  ## run flake8 on system test python files
+	flake8 system/
+
+lint: prepare
+	# Install golangci-lint
+	@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
+	# Running lint
+	@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0
+
+
+build: prepare swagger  ## Build aptly
+	go build -o build/aptly
+
+install:
+	@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
+	# go generate
+	@go generate
+	# go install -v
+	@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
+
+test: prepare swagger etcd-install  ## Run unit tests (add TEST=regex to specify which tests to run)
+	@echo "\e[33m\e[1mStarting etcd ...\e[0m"
+	@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
+	@echo "\e[33m\e[1mRunning go test ...\e[0m"
+	faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
+	@echo "\e[33m\e[1mStopping etcd ...\e[0m"
+	@pid=`cat /tmp/etcd.pid`; kill $$pid
+	@rm -f /tmp/aptly-etcd-data/etcd.log
+	@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
+
+system-test: prepare swagger etcd-install  ## Run system tests
+	# build coverage binary
+	go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
+	# Download fixture-db, fixture-pool, etcd.db
+	if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
+	if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
+	test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
+	# Run system tests
+	PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
+
+bench:
+	@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
+	go test -v ./deb -run=nothing -bench=. -benchmem
+
+serve: prepare swagger-install  ## Run development server (auto recompiling)
+	test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
+	cp debian/aptly.conf ~/.aptly.conf
+	sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
+	sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
+	PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
+
+dpkg: prepare swagger  ## Build debian packages
+	@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
+	# set debian version
+	@if [ "`make -s releasetype`" = "ci" ]; then  \
+		echo CI Build, setting version... ; \
+		test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog ; \
+		cp debian/changelog debian/changelog.dpkg-bak ; \
+		DEBEMAIL="CI <ci@aptly.info>" dch -v `make -s version` "CI build" ; \
+	fi
+	# clean
+	rm -rf obj-i686-linux-gnu obj-arm-linux-gnueabihf obj-aarch64-linux-gnu obj-x86_64-linux-gnu
+	# Run dpkg-buildpackage
+	@buildtype="any" ; \
+	if [ "$(DEBARCH)" = "amd64" ]; then  \
+	  buildtype="any,all" ; \
+	fi ; \
+	echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
+	cmd="dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)" ; \
+	echo "$$cmd" ; \
+	$$cmd
+	lintian ../*_$(DEBARCH).changes || true
+	# cleanup
+	@test ! -f debian/changelog.dpkg-bak || mv debian/changelog.dpkg-bak debian/changelog; \
+	mkdir -p build && mv ../*.deb build/ ; \
+	cd build && ls -l *.deb
+
+binaries: prepare swagger  ## Build binary releases (FreeBSD, macOS, Linux generic)
+	# build aptly
+	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
+	# install
+	@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
+	@cp man/aptly.1 build/tmp/man/
+	@cp completion.d/aptly build/tmp/completion/bash_completion.d/
+	@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
+	@cp README.rst LICENSE AUTHORS build/tmp/
+	@gzip -f build/tmp/man/aptly.1
+	@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
+	rm -rf "build/$$path"; \
+	mv build/tmp build/"$$path"; \
+	rm -rf build/tmp; \
+	cd build; \
+	zip -r "$$path".zip "$$path" > /dev/null \
+		&& echo "Built build/$${path}.zip"; \
+	rm -rf "$$path"
+
+docker-image:  ## Build aptly-dev docker image
+	@docker build -f system/Dockerfile . -t aptly-dev
+
+docker-image-no-cache:  ## Build aptly-dev docker image (no cache)
+	@docker build --no-cache -f system/Dockerfile . -t aptly-dev
+
+docker-build:  ## Build aptly in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
+
+docker-shell:  ## Run aptly and other commands in docker container
+	@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
+
+docker-deb:  ## Build debian packages in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
+
+docker-unit-test:  ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
+	$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		test TEST=$(TEST) \
+		azurite-stop
+
+docker-system-test:  ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
+		azurite-start \
+		AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
+		AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
+		AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
+		AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
+		AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
+		system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
+		azurite-stop
+
+docker-serve:  ## Run development server (auto recompiling) on http://localhost:3142
+	@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
+
+docker-lint:  ## Run golangci-lint in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
+
+docker-binaries:  ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
+
+docker-man:  ## Create man page in docker container
+	@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
+
+mem.png: mem.dat mem.gp
+	gnuplot mem.gp
+	open mem.png
+
+man:  ## Create man pages
+	make -C man
+
+clean:  ## remove local build and module cache
+	# Clean all generated and build files
+	test ! -e .go || find .go/ -type d ! -perm -u=w -exec chmod u+w {} \;
+	rm -rf .go/
+	rm -rf build/ obj-*-linux-gnu* tmp/
+	rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
+	find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
+
+.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8

README.rst

@@ -135,98 +135,3 @@ Scala sbt:
 Molior:
 
 -   `Molior Debian Build System <https://github.com/molior-dbs/molior>`_ by André Roth
-
-Configuration
-=============
-
-etcd Database Configuration
----------------------------
-
-When using etcd as the database backend, aptly supports several environment variables for configuration:
-
-**Timeout Configuration:**
-
-- ``APTLY_ETCD_TIMEOUT``: Operation timeout for etcd requests (default: ``60s``)
-  
-  Example: ``export APTLY_ETCD_TIMEOUT=30s``
-
-- ``APTLY_ETCD_DIAL_TIMEOUT``: Connection timeout when establishing etcd connection (default: ``60s``)
-  
-  Example: ``export APTLY_ETCD_DIAL_TIMEOUT=10s``
-
-**Connection Configuration:**
-
-- ``APTLY_ETCD_KEEPALIVE``: Keep-alive timeout for etcd connections (default: ``7200s``)
-  
-  Example: ``export APTLY_ETCD_KEEPALIVE=3600s``
-
-- ``APTLY_ETCD_MAX_MSG_SIZE``: Maximum message size in bytes for etcd requests/responses (default: ``52428800`` - 50MB)
-  
-  Example: ``export APTLY_ETCD_MAX_MSG_SIZE=104857600``  # 100MB
-
-**Example Configuration:**
-
-.. code-block:: bash
-
-    # Set shorter timeouts for faster failure detection
-    export APTLY_ETCD_TIMEOUT=30s
-    export APTLY_ETCD_DIAL_TIMEOUT=10s
-    
-    # Increase message size for large package operations
-    export APTLY_ETCD_MAX_MSG_SIZE=104857600
-    
-    # Run aptly with etcd backend
-    aptly -config=/etc/aptly-etcd.conf mirror update debian-stable
-
-**Features:**
-
-- **Automatic Retry**: Read operations (Get) automatically retry up to 3 times with exponential backoff on temporary failures
-- **Timeout Protection**: All etcd operations use context with timeout to prevent indefinite hangs
-- **Enhanced Logging**: All etcd errors are logged with operation context for better debugging
-- **Configurable Limits**: Message size limits can be adjusted for large package operations
-
-etcd Write Queue Configuration
-------------------------------
-
-To prevent etcd overload during concurrent operations (e.g., multiple mirror updates), aptly supports an optional write queue that serializes database write operations:
-
-**Configuration in aptly.conf:**
-
-.. code-block:: json
-
-    {
-      "databaseBackend": {
-        "type": "etcd",
-        "url": "localhost:2379",
-        "timeout": "120s",
-        "writeRetries": 3,
-        "writeQueue": {
-          "enabled": true,
-          "queueSize": 1000,
-          "maxWritesPerSec": 100,
-          "batchMaxSize": 50,
-          "batchMaxWaitMs": 10
-        }
-      }
-    }
-
-**Write Queue Options:**
-
-- ``enabled``: Enable/disable the write queue (default: ``false``)
-- ``queueSize``: Size of the write operation queue (default: ``1000``)
-- ``maxWritesPerSec``: Maximum write operations per second (default: ``100``)
-- ``batchMaxSize``: Maximum batch size for future batching support (default: ``50``)
-- ``batchMaxWaitMs``: Maximum wait time for batch accumulation in milliseconds (default: ``10``)
-
-**Benefits:**
-
-- **Prevents etcd Overload**: Serializes write operations to avoid overwhelming etcd
-- **Maintains Parallelism**: I/O operations like downloads remain parallel
-- **Rate Limiting**: Configurable writes per second to match etcd capacity
-- **Transparent**: No code changes required, just enable in configuration
-
-**Example Impact:**
-
-Without write queue: 5 mirror updates → 5 parallel writers → 1000s of concurrent etcd operations → timeouts
-
-With write queue: 5 mirror updates → 5 parallel processes → 1 sequential etcd writer → stable performance

api/api.go

@@ -7,7 +7,6 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 
 	"github.com/aptly-dev/aptly/aptly"
@@ -101,18 +100,7 @@ type dbRequest struct {
 	err  chan<- error
 }
 
-var (
-	dbRequests     chan dbRequest
-	dbRequestsOnce sync.Once
-)
-
-// initDBRequests initializes the database request channel in a thread-safe manner
-func initDBRequests() {
-	dbRequestsOnce.Do(func() {
-		dbRequests = make(chan dbRequest, 1)
-		go acquireDatabase()
-	})
-}
+var dbRequests chan dbRequest
 
 // Acquire database lock and release it when not needed anymore.
 //
@@ -151,8 +139,9 @@ func acquireDatabase() {
 // runTaskInBackground to run a task which accquire database.
 // Important do not forget to defer to releaseDatabaseConnection
 func acquireDatabaseConnection() error {
-	// Ensure channel is initialized
-	initDBRequests()
+	if dbRequests == nil {
+		return nil
+	}
 
 	errCh := make(chan error)
 	dbRequests <- dbRequest{acquiredb, errCh}
@@ -162,8 +151,9 @@ func acquireDatabaseConnection() error {
 
 // Release database connection when not needed anymore
 func releaseDatabaseConnection() error {
-	// Ensure channel is initialized
-	initDBRequests()
+	if dbRequests == nil {
+		return nil
+	}
 
 	errCh := make(chan error)
 	dbRequests <- dbRequest{releasedb, errCh}

api/api_packages_test.go

@@ -1,74 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-	"net/http/httptest"
-
-	"github.com/aptly-dev/aptly/deb"
-	"github.com/gin-gonic/gin"
-	. "gopkg.in/check.v1"
-)
-
-type ApiPackagesSuite struct {
-	APISuite
-}
-
-var _ = Suite(&ApiPackagesSuite{})
-
-func (s *ApiPackagesSuite) TestShowPackages(c *C) {
-	// Test showPackages function with nil reflist
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	showPackages(ginCtx, nil, s.context.NewCollectionFactory())
-	
-	// Should return 404 for nil reflist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *ApiPackagesSuite) TestShowPackagesWithEmptyList(c *C) {
-	// Test showPackages with empty package reflist
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	reflist := deb.NewPackageRefList()
-	showPackages(ginCtx, reflist, s.context.NewCollectionFactory())
-	
-	c.Check(w.Code, Equals, 200)
-	
-	var result []string
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(len(result), Equals, 0)
-}
-
-func (s *ApiPackagesSuite) TestShowPackagesCompact(c *C) {
-	// Test showPackages with compact format (default)
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	reflist := deb.NewPackageRefList()
-	showPackages(ginCtx, reflist, s.context.NewCollectionFactory())
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *ApiPackagesSuite) TestShowPackagesDetails(c *C) {
-	// Test showPackages with details format
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test?format=details", nil)
-	
-	reflist := deb.NewPackageRefList()
-	showPackages(ginCtx, reflist, s.context.NewCollectionFactory())
-	
-	c.Check(w.Code, Equals, 200)
-	
-	var result []*deb.Package
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(len(result), Equals, 0)
-}

api/api_test.go

@@ -13,8 +13,6 @@ import (
 
 	"github.com/aptly-dev/aptly/aptly"
 	ctx "github.com/aptly-dev/aptly/context"
-	"github.com/aptly-dev/aptly/deb"
-	"github.com/aptly-dev/aptly/task"
 	"github.com/gin-gonic/gin"
 
 	"github.com/smira/flag"
@@ -148,14 +146,8 @@ func (s *APISuite) TestRepoCreate(c *C) {
 		"Name": "dummy",
 	})
 	c.Assert(err, IsNil)
-	resp, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
+	_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
 	c.Assert(err, IsNil)
-	c.Check(resp.Code, Equals, 201)
-	
-	// Clean up: delete the created repo
-	resp, err = s.HTTPRequest("DELETE", "/api/repos/dummy?force=1", nil)
-	c.Assert(err, IsNil)
-	c.Check(resp.Code, Equals, 200)
 }
 
 func (s *APISuite) TestTruthy(c *C) {
@@ -181,178 +173,3 @@ func (s *APISuite) TestTruthy(c *C) {
 	c.Check(truthy(-1), Equals, true)
 	c.Check(truthy(gin.H{}), Equals, true)
 }
-
-func (s *APISuite) TestDatabaseConnectionFunctions(c *C) {
-	// Test acquire and release database connection
-	err := acquireDatabaseConnection()
-	c.Check(err, IsNil)
-	
-	err = releaseDatabaseConnection()
-	c.Check(err, IsNil)
-}
-
-func (s *APISuite) TestConcurrentDatabaseRequests(c *C) {
-	// Test concurrent database acquisition
-	done := make(chan bool, 5)
-	
-	for i := 0; i < 5; i++ {
-		go func() {
-			defer func() { done <- true }()
-			
-			err := acquireDatabaseConnection()
-			if err == nil {
-				_ = releaseDatabaseConnection()
-			}
-		}()
-	}
-	
-	// Wait for all goroutines
-	for i := 0; i < 5; i++ {
-		<-done
-	}
-	
-	c.Check(true, Equals, true) // If we get here, no deadlock occurred
-}
-
-func (s *APISuite) TestMaybeRunTaskInBackground(c *C) {
-	// Test synchronous task execution
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	called := false
-	maybeRunTaskInBackground(ginCtx, "test-task", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		called = true
-		return &task.ProcessReturnValue{Code: 200, Value: gin.H{"status": "ok"}}, nil
-	})
-	
-	c.Check(called, Equals, true)
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *APISuite) TestMaybeRunTaskInBackgroundAsync(c *C) {
-	// Test asynchronous task execution
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test?_async=true", nil)
-	
-	maybeRunTaskInBackground(ginCtx, "test-async-task", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return &task.ProcessReturnValue{Code: 200, Value: gin.H{"status": "ok"}}, nil
-	})
-	
-	// For async, should return 202 Accepted
-	c.Check(w.Code, Equals, 202)
-}
-
-func (s *APISuite) TestAbortWithJSONError(c *C) {
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	
-	testErr := fmt.Errorf("test error message")
-	AbortWithJSONError(ginCtx, 400, testErr)
-	
-	c.Check(w.Code, Equals, 400)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-}
-
-func (s *APISuite) TestShowPackagesWithNilList(c *C) {
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	showPackages(ginCtx, nil, s.context.NewCollectionFactory())
-	
-	// Should return error when reflist is nil
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *APISuite) TestAPIVersionConstant(c *C) {
-	// Test that apiVersion struct is properly defined
-	version := aptlyVersion{Version: "test-version"}
-	c.Check(version.Version, Equals, "test-version")
-}
-
-func (s *APISuite) TestAPIStatusConstant(c *C) {
-	// Test that aptlyStatus struct is properly defined
-	status := aptlyStatus{Status: "test-status"}
-	c.Check(status.Status, Equals, "test-status")
-}
-
-func (s *APISuite) TestRunTaskInBackground(c *C) {
-	// Test running task in background
-	task, err := runTaskInBackground("background-test", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return &task.ProcessReturnValue{Code: 200, Value: gin.H{"done": true}}, nil
-	})
-	
-	c.Check(err, IsNil)
-	c.Check(task, NotNil)
-	c.Check(task.Name, Equals, "background-test")
-	
-	// Wait for task to complete
-	_, _ = s.context.TaskList().WaitForTaskByID(task.ID)
-	
-	// Clean up
-	_, _ = s.context.TaskList().DeleteTaskByID(task.ID)
-}
-
-func (s *APISuite) TestInitDBRequests(c *C) {
-	// Test that initDBRequests can be called multiple times safely
-	initDBRequests()
-	initDBRequests() // Should not panic
-	
-	c.Check(dbRequests, NotNil)
-}
-
-func (s *APISuite) TestShowPackagesWithQuery(c *C) {
-	// Create a test gin context
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test?q=Name&format=details", nil)
-	
-	// Create empty reflist
-	reflist := deb.NewPackageRefList()
-	
-	showPackages(ginCtx, reflist, s.context.NewCollectionFactory())
-	
-	// Should succeed with empty list
-	c.Check(w.Code, Equals, 200)
-	
-	var result []*deb.Package
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(len(result), Equals, 0)
-}
-
-func (s *APISuite) TestShowPackagesCompactFormat(c *C) {
-	// Test compact format (default)
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	reflist := deb.NewPackageRefList()
-	showPackages(ginCtx, reflist, s.context.NewCollectionFactory())
-	
-	c.Check(w.Code, Equals, 200)
-	
-	var result []string
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(len(result), Equals, 0)
-}
-
-func (s *APISuite) TestTruthyEdgeCases(c *C) {
-	// Test edge cases for truthy function
-	c.Check(truthy("F"), Equals, false) // capital F
-	c.Check(truthy("FALSE"), Equals, false) // all caps
-	c.Check(truthy("False"), Equals, false) // mixed case
-	c.Check(truthy("NO"), Equals, false) // capital NO
-	c.Check(truthy("Off"), Equals, false) // mixed case off
-	
-	// Test empty string
-	c.Check(truthy(""), Equals, true) // empty string is truthy
-	
-	// Test other types
-	c.Check(truthy(struct{}{}), Equals, true) // empty struct
-	c.Check(truthy([]int{}), Equals, true) // empty slice
-	c.Check(truthy(map[string]int{}), Equals, true) // empty map
-}

api/db_test.go

@@ -1,362 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"net/http"
-	"net/http/httptest"
-	"time"
-
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/task"
-	"github.com/gin-gonic/gin"
-	. "gopkg.in/check.v1"
-)
-
-type DBTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&DBTestSuite{})
-
-func (s *DBTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *DBTestSuite) TestDbCleanupStructure(c *C) {
-	// Test database cleanup endpoint structure
-	req, _ := http.NewRequest("POST", "/api/db/cleanup", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should succeed with proper context
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *DBTestSuite) TestDbCleanupWithAsync(c *C) {
-	// Test database cleanup with async parameter
-	req, _ := http.NewRequest("POST", "/api/db/cleanup?_async=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should return task response when async
-	c.Check(w.Code, Equals, 202)
-}
-
-func (s *DBTestSuite) TestDbCleanupWithDryRun(c *C) {
-	// Test database cleanup with dry run parameter
-	req, _ := http.NewRequest("POST", "/api/db/cleanup?dry-run=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should succeed with dry run
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *DBTestSuite) TestDbCleanupWithBothParams(c *C) {
-	// Test database cleanup with both async and dry-run parameters
-	req, _ := http.NewRequest("POST", "/api/db/cleanup?_async=1&dry-run=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests parameter combination
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *DBTestSuite) TestDbCleanupHTTPMethods(c *C) {
-	// Test that only POST method is allowed
-	deniedMethods := []string{"GET", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
-
-	for _, method := range deniedMethods {
-		req, _ := http.NewRequest(method, "/api/db/cleanup", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 404, Commentf("Method: %s should be denied", method))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupWithRequestBody(c *C) {
-	// Test database cleanup with various request bodies (should be ignored)
-	testBodies := []string{
-		"",
-		"some random text",
-		`{"key": "value"}`,
-		`<xml>data</xml>`,
-		"binary\x00\x01\x02data",
-	}
-
-	for i, body := range testBodies {
-		req, _ := http.NewRequest("POST", "/api/db/cleanup", bytes.NewBufferString(body))
-		req.Header.Set("Content-Type", "application/json")
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle various body content without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Body test #%d", i+1))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupParameterVariations(c *C) {
-	// Test various parameter value combinations
-	paramTests := []struct {
-		query       string
-		description string
-	}{
-		{"", "no parameters"},
-		{"_async=0", "async disabled"},
-		{"_async=false", "async false"},
-		{"_async=true", "async true"},
-		{"dry-run=0", "dry-run disabled"},
-		{"dry-run=false", "dry-run false"},
-		{"dry-run=true", "dry-run true"},
-		{"_async=1&dry-run=0", "async on, dry-run off"},
-		{"_async=0&dry-run=1", "async off, dry-run on"},
-		{"_async=true&dry-run=false", "async true, dry-run false"},
-		{"unknown=param", "unknown parameter"},
-		{"_async=invalid", "invalid async value"},
-		{"dry-run=invalid", "invalid dry-run value"},
-	}
-
-	for _, test := range paramTests {
-		path := "/api/db/cleanup"
-		if test.query != "" {
-			path += "?" + test.query
-		}
-
-		req, _ := http.NewRequest("POST", path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle all parameter variations without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", test.description))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupContentTypes(c *C) {
-	// Test different content types
-	contentTypes := []string{
-		"",
-		"application/json",
-		"text/plain",
-		"application/x-www-form-urlencoded",
-		"multipart/form-data",
-		"application/octet-stream",
-	}
-
-	for _, contentType := range contentTypes {
-		req, _ := http.NewRequest("POST", "/api/db/cleanup", nil)
-		if contentType != "" {
-			req.Header.Set("Content-Type", contentType)
-		}
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle different content types without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Content-Type: %s", contentType))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupErrorHandling(c *C) {
-	// Test various error conditions
-	errorTests := []struct {
-		description string
-		path        string
-		method      string
-		expectError bool
-	}{
-		{"Normal cleanup call", "/api/db/cleanup", "POST", true},            // Expect error due to no context
-		{"Cleanup with extra path", "/api/db/cleanup/extra", "POST", false}, // Route not matched
-		{"Cleanup normal path", "/api/db/cleanup", "POST", true},  // Valid endpoint
-		{"Case sensitive path", "/api/DB/cleanup", "POST", false},           // Route not matched
-		{"Case sensitive path", "/api/db/CLEANUP", "POST", false},           // Route not matched
-	}
-
-	for _, test := range errorTests {
-		req, _ := http.NewRequest(test.method, test.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// All should return some response without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", test.description))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupReliability(c *C) {
-	// Test multiple sequential calls for reliability
-	for i := 0; i < 5; i++ {
-		req, _ := http.NewRequest("POST", "/api/db/cleanup", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should be consistent across multiple calls
-		c.Check(w.Code, Not(Equals), 0, Commentf("Call #%d", i+1))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupHeaders(c *C) {
-	// Test with various HTTP headers
-	headerTests := []map[string]string{
-		{},
-		{"Accept": "application/json"},
-		{"Accept": "text/plain"},
-		{"Accept": "*/*"},
-		{"User-Agent": "test-agent"},
-		{"Authorization": "Bearer token123"},
-		{"X-Custom-Header": "custom-value"},
-		{"Accept-Encoding": "gzip, deflate"},
-		{"Accept-Language": "en-US,en;q=0.9"},
-	}
-
-	for i, headers := range headerTests {
-		req, _ := http.NewRequest("POST", "/api/db/cleanup", nil)
-		for key, value := range headers {
-			req.Header.Set(key, value)
-		}
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle various headers without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Header test #%d", i+1))
-	}
-}
-
-func (s *DBTestSuite) TestDbCleanupResponseFormat(c *C) {
-	// Test response format consistency
-	req, _ := http.NewRequest("POST", "/api/db/cleanup", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should have proper response structure
-	c.Check(w.Code, Not(Equals), 0)
-	c.Check(w.Header(), NotNil)
-
-	// If there's a response body, it should be valid
-	if w.Body.Len() > 0 {
-		body := w.Body.String()
-		c.Check(len(body), Not(Equals), 0)
-	}
-}
-
-func (s *DBTestSuite) TestDbRequestTypes(c *C) {
-	// Test dbRequestKind constants
-	c.Check(acquiredb, Equals, dbRequestKind(0))
-	c.Check(releasedb, Equals, dbRequestKind(1))
-}
-
-func (s *DBTestSuite) TestDbRequestStruct(c *C) {
-	// Test dbRequest struct creation
-	errCh := make(chan error, 1)
-	req := dbRequest{
-		kind: acquiredb,
-		err:  errCh,
-	}
-	
-	c.Check(req.kind, Equals, acquiredb)
-	c.Check(req.err, NotNil)
-}
-
-func (s *DBTestSuite) TestAcquireAndReleaseDatabase(c *C) {
-	// Initialize db requests channel
-	initDBRequests()
-	
-	// Test multiple acquire and release cycles
-	for i := 0; i < 3; i++ {
-		err := acquireDatabaseConnection()
-		c.Check(err, IsNil)
-		
-		err = releaseDatabaseConnection()
-		c.Check(err, IsNil)
-	}
-}
-
-func (s *DBTestSuite) TestConcurrentDatabaseAccess(c *C) {
-	// Test concurrent database access
-	done := make(chan bool, 10)
-	
-	for i := 0; i < 10; i++ {
-		go func(id int) {
-			defer func() { done <- true }()
-			
-			// Acquire and release database connection
-			if err := acquireDatabaseConnection(); err == nil {
-				// Simulate some work
-				time.Sleep(10 * time.Millisecond)
-				_ = releaseDatabaseConnection()
-			}
-		}(i)
-	}
-	
-	// Wait for all goroutines to complete
-	for i := 0; i < 10; i++ {
-		<-done
-	}
-	
-	c.Check(true, Equals, true) // Test passed without deadlock
-}
-
-func (s *DBTestSuite) TestMaybeRunTaskInBackgroundWithError(c *C) {
-	// Test task that returns an error
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	testErr := gin.Error{Type: gin.ErrorTypePublic, Err: gin.Error{}.Err}
-	maybeRunTaskInBackground(ginCtx, "error-task", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return nil, testErr
-	})
-	
-	// Should return error status
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *DBTestSuite) TestMaybeRunTaskInBackgroundConflict(c *C) {
-	// Test task with resource conflict
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	// Create two tasks with same resources to cause conflict
-	resource := "test-resource-" + time.Now().Format("20060102150405")
-	
-	// Start first task
-	_, _ = runTaskInBackground("task1", []string{resource}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		time.Sleep(100 * time.Millisecond) // Hold resource
-		return &task.ProcessReturnValue{Code: 200}, nil
-	})
-	
-	// Try to start second task with same resource (should conflict)
-	maybeRunTaskInBackground(ginCtx, "task2", []string{resource}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return &task.ProcessReturnValue{Code: 200}, nil
-	})
-	
-	// Should return 409 Conflict
-	c.Check(w.Code, Equals, 409)
-}
-
-func (s *DBTestSuite) TestRunTaskInBackgroundWithNilReturn(c *C) {
-	// Test task that returns nil ProcessReturnValue
-	task, err := runTaskInBackground("nil-return-task", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return nil, nil
-	})
-	
-	c.Check(err, IsNil)
-	c.Check(task, NotNil)
-	
-	// Wait and clean up
-	_, _ = s.context.TaskList().WaitForTaskByID(task.ID)
-	_, _ = s.context.TaskList().DeleteTaskByID(task.ID)
-}
-
-func (s *DBTestSuite) TestMaybeRunTaskInBackgroundNilReturn(c *C) {
-	// Test synchronous task with nil return value
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/test", nil)
-	
-	maybeRunTaskInBackground(ginCtx, "nil-sync-task", []string{}, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
-		return nil, nil
-	})
-	
-	// Should return 200 with nil body
-	c.Check(w.Code, Equals, 200)
-}

api/error_test.go

@@ -1,267 +0,0 @@
-package api
-
-import (
-	"encoding/json"
-
-	. "gopkg.in/check.v1"
-)
-
-type ErrorTestSuite struct{}
-
-var _ = Suite(&ErrorTestSuite{})
-
-func (s *ErrorTestSuite) TestErrorStruct(c *C) {
-	// Test Error struct creation and fields
-	err := Error{Error: "test error message"}
-	c.Check(err.Error, Equals, "test error message")
-}
-
-func (s *ErrorTestSuite) TestErrorJSONMarshaling(c *C) {
-	// Test JSON marshaling of Error struct
-	err := Error{Error: "test error message"}
-
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-	c.Check(string(jsonData), Equals, `{"error":"test error message"}`)
-}
-
-func (s *ErrorTestSuite) TestErrorJSONUnmarshaling(c *C) {
-	// Test JSON unmarshaling into Error struct
-	jsonData := `{"error":"test error message"}`
-
-	var err Error
-	unmarshalErr := json.Unmarshal([]byte(jsonData), &err)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(err.Error, Equals, "test error message")
-}
-
-func (s *ErrorTestSuite) TestErrorEmptyMessage(c *C) {
-	// Test Error struct with empty message
-	err := Error{Error: ""}
-	c.Check(err.Error, Equals, "")
-
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-	c.Check(string(jsonData), Equals, `{"error":""}`)
-}
-
-func (s *ErrorTestSuite) TestErrorSpecialCharacters(c *C) {
-	// Test Error struct with special characters
-	specialMessages := []string{
-		"error with \"quotes\"",
-		"error with 'apostrophes'",
-		"error with \n newlines",
-		"error with \t tabs",
-		"error with unicode: 你好",
-		"error with emoji: 🚨❌",
-		"error with backslashes: \\path\\to\\file",
-		"error with json characters: {\"key\": \"value\"}",
-		"error with < > & characters",
-		"error with null \x00 character",
-	}
-
-	for i, message := range specialMessages {
-		err := Error{Error: message}
-		c.Check(err.Error, Equals, message, Commentf("Test case %d", i))
-
-		// Test JSON marshaling works with special characters
-		jsonData, marshalErr := json.Marshal(err)
-		c.Check(marshalErr, IsNil, Commentf("Marshal failed for case %d: %s", i, message))
-
-		// Test JSON unmarshaling works with special characters
-		var unmarshaled Error
-		unmarshalErr := json.Unmarshal(jsonData, &unmarshaled)
-		c.Check(unmarshalErr, IsNil, Commentf("Unmarshal failed for case %d: %s", i, message))
-		c.Check(unmarshaled.Error, Equals, message, Commentf("Round-trip failed for case %d", i))
-	}
-}
-
-func (s *ErrorTestSuite) TestErrorLongMessage(c *C) {
-	// Test Error struct with very long message
-	longMessage := ""
-	for i := 0; i < 1000; i++ {
-		longMessage += "This is a very long error message. "
-	}
-
-	err := Error{Error: longMessage}
-	c.Check(err.Error, Equals, longMessage)
-
-	// Test JSON marshaling/unmarshaling with long message
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-
-	var unmarshaled Error
-	unmarshalErr := json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(unmarshaled.Error, Equals, longMessage)
-}
-
-func (s *ErrorTestSuite) TestErrorJSONFieldName(c *C) {
-	// Test that the JSON field name is exactly "error"
-	err := Error{Error: "test"}
-
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-
-	// Parse as generic map to check field name
-	var result map[string]interface{}
-	unmarshalErr := json.Unmarshal(jsonData, &result)
-	c.Check(unmarshalErr, IsNil)
-
-	// Check that the field is named "error"
-	value, exists := result["error"]
-	c.Check(exists, Equals, true)
-	c.Check(value, Equals, "test")
-
-	// Check that no other fields exist
-	c.Check(len(result), Equals, 1)
-}
-
-func (s *ErrorTestSuite) TestErrorJSONWithExtraFields(c *C) {
-	// Test unmarshaling JSON with extra fields (should be ignored)
-	jsonData := `{"error":"test error","extra":"ignored","number":123}`
-
-	var err Error
-	unmarshalErr := json.Unmarshal([]byte(jsonData), &err)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(err.Error, Equals, "test error")
-}
-
-func (s *ErrorTestSuite) TestErrorJSONMissingField(c *C) {
-	// Test unmarshaling JSON missing the error field
-	jsonData := `{"other":"value"}`
-
-	var err Error
-	unmarshalErr := json.Unmarshal([]byte(jsonData), &err)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(err.Error, Equals, "") // Should be zero value
-}
-
-func (s *ErrorTestSuite) TestErrorJSONInvalidJSON(c *C) {
-	// Test unmarshaling invalid JSON
-	invalidJSONs := []string{
-		`{"error":}`,
-		`{"error": invalid}`,
-		`{error: "missing quotes"}`,
-		`{"error": "unterminated`,
-		`malformed json`,
-		``,
-		`null`,
-		`[]`,
-		`123`,
-	}
-
-	for i, jsonData := range invalidJSONs {
-		var err Error
-		unmarshalErr := json.Unmarshal([]byte(jsonData), &err)
-
-		// Should either error or handle gracefully
-		if unmarshalErr == nil {
-			// If no error, check the result is reasonable
-			c.Check(err.Error, FitsTypeOf, "", Commentf("Invalid JSON case %d: %s", i, jsonData))
-		} else {
-			// Error is expected for malformed JSON
-			c.Check(unmarshalErr, NotNil, Commentf("Expected error for case %d: %s", i, jsonData))
-		}
-	}
-}
-
-func (s *ErrorTestSuite) TestErrorZeroValue(c *C) {
-	// Test zero value of Error struct
-	var err Error
-	c.Check(err.Error, Equals, "")
-
-	// Test JSON marshaling of zero value
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-	c.Check(string(jsonData), Equals, `{"error":""}`)
-}
-
-func (s *ErrorTestSuite) TestErrorPointer(c *C) {
-	// Test Error struct as pointer
-	err := &Error{Error: "pointer error"}
-	c.Check(err.Error, Equals, "pointer error")
-
-	// Test JSON marshaling of pointer
-	jsonData, marshalErr := json.Marshal(err)
-	c.Check(marshalErr, IsNil)
-	c.Check(string(jsonData), Equals, `{"error":"pointer error"}`)
-
-	// Test JSON unmarshaling into pointer
-	var err2 *Error
-	unmarshalErr := json.Unmarshal(jsonData, &err2)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(err2, NotNil)
-	c.Check(err2.Error, Equals, "pointer error")
-}
-
-func (s *ErrorTestSuite) TestErrorStructCopy(c *C) {
-	// Test copying Error struct
-	err1 := Error{Error: "original error"}
-	err2 := err1
-
-	c.Check(err2.Error, Equals, "original error")
-
-	// Modify original and ensure copy is independent
-	err1.Error = "modified error"
-	c.Check(err1.Error, Equals, "modified error")
-	c.Check(err2.Error, Equals, "original error")
-}
-
-func (s *ErrorTestSuite) TestErrorStructComparison(c *C) {
-	// Test comparing Error structs
-	err1 := Error{Error: "same message"}
-	err2 := Error{Error: "same message"}
-	err3 := Error{Error: "different message"}
-
-	c.Check(err1 == err2, Equals, true)
-	c.Check(err1 == err3, Equals, false)
-	c.Check(err2 == err3, Equals, false)
-}
-
-func (s *ErrorTestSuite) TestErrorStructInSlice(c *C) {
-	// Test Error struct in slice operations
-	errors := []Error{
-		{Error: "first error"},
-		{Error: "second error"},
-		{Error: "third error"},
-	}
-
-	c.Check(len(errors), Equals, 3)
-	c.Check(errors[0].Error, Equals, "first error")
-	c.Check(errors[1].Error, Equals, "second error")
-	c.Check(errors[2].Error, Equals, "third error")
-
-	// Test JSON marshaling of slice
-	jsonData, marshalErr := json.Marshal(errors)
-	c.Check(marshalErr, IsNil)
-
-	var unmarshaled []Error
-	unmarshalErr := json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(len(unmarshaled), Equals, 3)
-	c.Check(unmarshaled[0].Error, Equals, "first error")
-}
-
-func (s *ErrorTestSuite) TestErrorStructInMap(c *C) {
-	// Test Error struct in map operations
-	errorMap := map[string]Error{
-		"key1": {Error: "first error"},
-		"key2": {Error: "second error"},
-	}
-
-	c.Check(len(errorMap), Equals, 2)
-	c.Check(errorMap["key1"].Error, Equals, "first error")
-	c.Check(errorMap["key2"].Error, Equals, "second error")
-
-	// Test JSON marshaling of map
-	jsonData, marshalErr := json.Marshal(errorMap)
-	c.Check(marshalErr, IsNil)
-
-	var unmarshaled map[string]Error
-	unmarshalErr := json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(unmarshalErr, IsNil)
-	c.Check(len(unmarshaled), Equals, 2)
-	c.Check(unmarshaled["key1"].Error, Equals, "first error")
-	c.Check(unmarshaled["key2"].Error, Equals, "second error")
-}

api/files.go

@@ -13,6 +13,10 @@ import (
 	"github.com/saracen/walker"
 )
 
+// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
+// In production it calls (*os.File).Sync().
+var syncFile = func(f *os.File) error { return f.Sync() }
+
 func verifyPath(path string) bool {
 	path = filepath.Clean(path)
 	for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -114,34 +118,69 @@ func apiFilesUpload(c *gin.Context) {
 	}
 
 	stored := []string{}
+	openFiles := []*os.File{}
 
+	// Write all files first
 	for _, files := range c.Request.MultipartForm.File {
 		for _, file := range files {
 			src, err := file.Open()
 			if err != nil {
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer func() { _ = src.Close() }()
 
 			destPath := filepath.Join(path, filepath.Base(file.Filename))
 			dst, err := os.Create(destPath)
 			if err != nil {
+				_ = src.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
-			defer func() { _ = dst.Close() }()
 
 			_, err = io.Copy(dst, src)
+			_ = src.Close()
 			if err != nil {
+				_ = dst.Close()
+				// Close any files we've opened
+				for _, f := range openFiles {
+					_ = f.Close()
+				}
 				AbortWithJSONError(c, 500, err)
 				return
 			}
 
+			// Keep file open for batch sync
+			openFiles = append(openFiles, dst)
 			stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
 		}
 	}
 
+	// Sync all files at once to catch ENOSPC errors
+	for i, dst := range openFiles {
+		err := syncFile(dst)
+		if err != nil {
+			// Close all files
+			for _, f := range openFiles {
+				_ = f.Close()
+			}
+			AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
+			return
+		}
+	}
+
+	// Close all files
+	for _, dst := range openFiles {
+		_ = dst.Close()
+	}
+
 	apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
 	c.JSON(200, stored)
 }

api/files_test.go

@@ -3,336 +3,474 @@ package api
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
+	"io"
 	"mime/multipart"
+	"net/http"
 	"net/http/httptest"
 	"os"
 	"path/filepath"
-	"sync/atomic"
-	"testing"
+	"strings"
+	"syscall"
 
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
 	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
 	. "gopkg.in/check.v1"
 )
 
-// Hook up gocheck into the "go test" runner.
-func TestFiles(t *testing.T) { TestingT(t) }
-
-type FilesSuite struct {
-	APISuite
+type FilesUploadDiskFullSuite struct {
+	aptlyContext *ctx.AptlyContext
+	flags        *flag.FlagSet
+	configFile   *os.File
+	router       http.Handler
 }
 
-var _ = Suite(&FilesSuite{})
+var _ = Suite(&FilesUploadDiskFullSuite{})
 
-func (s *FilesSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
+func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
+	aptly.Version = "testVersion"
 
-func (s *FilesSuite) TearDownTest(c *C) {
-	// Clean up any test files
-	if s.context != nil {
-		uploadPath := s.context.UploadPath()
-		if uploadPath != "" {
-			os.RemoveAll(uploadPath)
-		}
-	}
-	s.APISuite.TearDownTest(c)
-}
-
-func (s *FilesSuite) TestVerifyPath(c *C) {
-	// Valid paths
-	c.Check(verifyPath("valid-dir"), Equals, true)
-	c.Check(verifyPath("valid/sub/dir"), Equals, true)
-	c.Check(verifyPath("valid/../other"), Equals, true)  // filepath.Clean normalizes to "other"
-
-	// Invalid paths
-	c.Check(verifyPath(""), Equals, false)  // Empty path becomes "."
-	c.Check(verifyPath("../invalid"), Equals, false)  // Contains ".."
-	c.Check(verifyPath(".."), Equals, false)  // Is ".."
-	c.Check(verifyPath("."), Equals, false)  // Is "."
-	c.Check(verifyPath("./"), Equals, false)  // Contains "."
-}
-
-func (s *FilesSuite) TestVerifyDirValid(c *C) {
-	// Create a test gin context
-	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Params = gin.Params{
-		{Key: "dir", Value: "valid-dir"},
-	}
-
-	result := verifyDir(ctx)
-	c.Check(result, Equals, true)
-}
-
-func (s *FilesSuite) TestVerifyDirInvalid(c *C) {
-	// Create a test gin context
-	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Params = gin.Params{
-		{Key: "dir", Value: "../invalid"},
-	}
-
-	result := verifyDir(ctx)
-	c.Check(result, Equals, false)
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *FilesSuite) TestApiFilesListDirs(c *C) {
-	// Create upload directory for testing
-	uploadPath := s.context.UploadPath()
-	err := os.MkdirAll(filepath.Join(uploadPath, "test-dir"), 0755)
+	file, err := os.CreateTemp("", "aptly")
 	c.Assert(err, IsNil)
-	defer os.RemoveAll(uploadPath)
-	
-	// Create test file
-	f, err := os.Create(filepath.Join(uploadPath, "test-file.txt"))
-	c.Assert(err, IsNil)
-	f.Close()
+	s.configFile = file
 
-	// Create test request
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("GET", "/api/files", nil)
-	s.router.ServeHTTP(w, req)
-
-	// Check response
-	c.Check(w.Code, Equals, 200)
-	var result []string
-	err = json.Unmarshal(w.Body.Bytes(), &result)
+	jsonString, err := json.Marshal(gin.H{
+		"architectures": []string{},
+		"rootDir":       c.MkDir(),
+	})
 	c.Assert(err, IsNil)
-	c.Check(len(result), Equals, 1)
-	c.Check(result[0], Equals, "test-dir")
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+	_ = file.Close()
+
+	flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", false, "dummy")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "dummy")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	aptlyContext, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.aptlyContext = aptlyContext
+	s.router = Router(aptlyContext)
+	context = aptlyContext
 }
 
-func (s *FilesSuite) TestApiFilesUpload(c *C) {
-	// Create multipart form data
+func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.aptlyContext != nil {
+		s.aptlyContext.Shutdown()
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
+	testContent := []byte("test file content for upload")
+
 	body := &bytes.Buffer{}
 	writer := multipart.NewWriter(body)
-	part, err := writer.CreateFormFile("file", "test.txt")
-	c.Assert(err, IsNil)
-	part.Write([]byte("test content"))
-	writer.Close()
 
-	// Create test request
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("POST", "/api/files/testdir", body)
+	part, err := writer.CreateFormFile("file", "testfile.txt")
+	c.Assert(err, IsNil)
+
+	_, err = part.Write(testContent)
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
 	req.Header.Set("Content-Type", writer.FormDataContentType())
-	s.router.ServeHTTP(w, req)
 
-	// Check response
-	c.Check(w.Code, Equals, 200)
-	
-	// Verify file was uploaded
-	uploadPath := filepath.Join(s.context.UploadPath(), "testdir", "test.txt")
-	_, err = os.Stat(uploadPath)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
+	content, err := os.ReadFile(uploadedFile)
 	c.Assert(err, IsNil)
-	
-	// Clean up
-	os.RemoveAll(filepath.Join(s.context.UploadPath(), "testdir"))
+	c.Check(content, DeepEquals, testContent)
 }
 
-func (s *FilesSuite) TestApiFilesListFiles(c *C) {
-	// Create test directory and files
-	testDir := filepath.Join(s.context.UploadPath(), "testdir")
-	err := os.MkdirAll(testDir, 0755)
+func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
+	testContent := bytes.Repeat([]byte("A"), 10000)
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("file", "largefile.bin")
 	c.Assert(err, IsNil)
-	
-	// Create test files
-	for i := 0; i < 3; i++ {
-		f, err := os.Create(filepath.Join(testDir, fmt.Sprintf("test%d.txt", i)))
+
+	_, err = io.Copy(part, bytes.NewReader(testContent))
+	c.Assert(err, IsNil)
+
+	err = writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/testdir2", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
+	content, err := os.ReadFile(uploadedFile)
+	c.Assert(err, IsNil)
+	c.Check(len(content), Equals, len(testContent))
+	c.Check(content, DeepEquals, testContent)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
+	testFiles := map[string][]byte{
+		"file1.txt": []byte("content of file 1"),
+		"file2.txt": bytes.Repeat([]byte("B"), 5000),
+		"file3.deb": []byte("debian package content"),
+	}
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	for filename, content := range testFiles {
+		part, err := writer.CreateFormFile("file", filename)
+		c.Assert(err, IsNil)
+		_, err = part.Write(content)
 		c.Assert(err, IsNil)
-		f.Close()
-	}
-	
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("GET", "/api/files/testdir", nil)
-	s.router.ServeHTTP(w, req)
-
-	// Check response
-	c.Check(w.Code, Equals, 200)
-	
-	var result []string
-	err = json.Unmarshal(w.Body.Bytes(), &result)
-	c.Assert(err, IsNil)
-	c.Check(len(result), Equals, 3)
-	
-	// Clean up
-	os.RemoveAll(testDir)
-}
-
-func (s *FilesSuite) TestApiFilesDeleteDir(c *C) {
-	// Create test directory
-	testDir := filepath.Join(s.context.UploadPath(), "testdir")
-	err := os.MkdirAll(testDir, 0755)
-	c.Assert(err, IsNil)
-	
-	// Create test file in directory
-	f, err := os.Create(filepath.Join(testDir, "test.txt"))
-	c.Assert(err, IsNil)
-	f.Close()
-	
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("DELETE", "/api/files/testdir", nil)
-	s.router.ServeHTTP(w, req)
-
-	// Check response
-	c.Check(w.Code, Equals, 200)
-	
-	// Verify directory was deleted
-	_, err = os.Stat(testDir)
-	c.Assert(os.IsNotExist(err), Equals, true)
-}
-
-func (s *FilesSuite) TestApiFilesDeleteFile(c *C) {
-	// Create test directory and file
-	testDir := filepath.Join(s.context.UploadPath(), "testdir")
-	err := os.MkdirAll(testDir, 0755)
-	c.Assert(err, IsNil)
-	
-	testFile := filepath.Join(testDir, "test.txt")
-	f, err := os.Create(testFile)
-	c.Assert(err, IsNil)
-	f.Write([]byte("test content"))
-	f.Close()
-	
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("DELETE", "/api/files/testdir/test.txt", nil)
-	s.router.ServeHTTP(w, req)
-
-	// Check response
-	c.Check(w.Code, Equals, 200)
-	
-	// Verify file was deleted
-	_, err = os.Stat(testFile)
-	c.Assert(os.IsNotExist(err), Equals, true)
-	
-	// Clean up
-	os.RemoveAll(testDir)
-}
-
-func (s *FilesSuite) TestApiFilesDeleteFileInvalidPath(c *C) {
-	// Create test request with invalid path
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("DELETE", "/api/files/testdir/../invalid", nil)
-	s.router.ServeHTTP(w, req)
-
-	// Should reject with 404 (not found) or 400 (bad request)
-	c.Check(w.Code == 400 || w.Code == 404, Equals, true)
-}
-
-// Custom checker for file existence
-var testFileExists Checker = &fileExistsChecker{
-	CheckerInfo: &CheckerInfo{Name: "testFileExists", Params: []string{"filename"}},
-}
-
-type fileExistsChecker struct {
-	*CheckerInfo
-}
-
-func (checker *fileExistsChecker) Check(params []interface{}, names []string) (result bool, error string) {
-	filename, ok := params[0].(string)
-	if !ok {
-		return false, "filename must be a string"
 	}
 
-	_, err := os.Stat(filename)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return false, ""
+	err := writer.Close()
+	c.Assert(err, IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/multitest", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+
+	uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
+	for filename, expectedContent := range testFiles {
+		uploadedFile := filepath.Join(uploadDir, filename)
+		content, err := os.ReadFile(uploadedFile)
+		c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
+		c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
+	}
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
+	oldSyncFile := syncFile
+	syncFile = func(f *os.File) error {
+		if filepath.Base(f.Name()) == "syncfail.txt" {
+			return syscall.ENOSPC
 		}
-		return false, err.Error()
+		return nil
 	}
-	return true, ""
-}
+	defer func() { syncFile = oldSyncFile }()
 
-// Test core API functions
-func (s *FilesSuite) TestApiVersion(c *C) {
-	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Request = httptest.NewRequest("GET", "/api/version", nil)
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
 
-	apiVersion(ctx)
+	part1, err := writer.CreateFormFile("file", "ok.txt")
+	c.Assert(err, IsNil)
+	_, err = part1.Write([]byte("ok"))
+	c.Assert(err, IsNil)
 
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Body.String(), Matches, `.*"Version":.*`)
-}
+	part2, err := writer.CreateFormFile("file", "syncfail.txt")
+	c.Assert(err, IsNil)
+	_, err = part2.Write([]byte("will fail on sync"))
+	c.Assert(err, IsNil)
 
-func (s *FilesSuite) TestApiHealthy(c *C) {
-	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Request = httptest.NewRequest("GET", "/api/healthy", nil)
+	err = writer.Close()
+	c.Assert(err, IsNil)
 
-	apiHealthy(ctx)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Body.String(), Matches, `.*"Status":"Aptly is healthy".*`)
-}
-
-func (s *FilesSuite) TestApiReadyWhenReady(c *C) {
-	isReady := &atomic.Value{}
-	isReady.Store(true)
+	req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
 
 	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Request = httptest.NewRequest("GET", "/api/ready", nil)
+	s.router.ServeHTTP(w, req)
 
-	apiReady(isReady)(ctx)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Body.String(), Matches, `.*"Status":"Aptly is ready".*`)
+	c.Assert(w.Code, Equals, 500)
+	c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
 }
 
-func (s *FilesSuite) TestApiReadyWhenNotReady(c *C) {
-	isReady := &atomic.Value{}
-	isReady.Store(false)
+func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
+	c.Check(verifyPath("a/b/c"), Equals, true)
+	c.Check(verifyPath("../x"), Equals, false)
+	c.Check(verifyPath("./x"), Equals, true)
+	c.Check(verifyPath(".."), Equals, false)
+	c.Check(verifyPath("."), Equals, false)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
+	uploadRoot := s.aptlyContext.UploadPath()
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
+	c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "d1"), Equals, true)
+	c.Check(strings.Contains(body, "d2"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
+	req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 404)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files/dir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	body := w.Body.String()
+	c.Check(strings.Contains(body, "a.txt"), Equals, true)
+	c.Check(strings.Contains(body, "b.txt"), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(base)
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+
+	_, statErr := os.Stat(filepath.Join(base, "a.txt"))
+	c.Check(os.IsNotExist(statErr), Equals, true)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 200)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
+	req, err := http.NewRequest("DELETE", "/api/files/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
+}
+
+func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
+
+	req, err := http.NewRequest("GET", "/api/files", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 200)
+	c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
+}
+
+func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+	c.Assert(os.Chmod(base, 0), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	req, err := http.NewRequest("GET", "/api/files/noperms", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	subdir := filepath.Join(base, "subdir")
+	c.Assert(os.MkdirAll(subdir, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
+
+	req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
+	req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
 
 	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Request = httptest.NewRequest("GET", "/api/ready", nil)
+	s.router.ServeHTTP(w, req)
 
-	apiReady(isReady)(ctx)
-
-	c.Check(w.Code, Equals, 503)
-	c.Check(w.Body.String(), Matches, `.*"Status":"Aptly is unavailable".*`)
+	c.Assert(w.Code, Equals, 400)
 }
 
-func (s *FilesSuite) TestApiReadyWithNil(c *C) {
+func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/..", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
 	w := httptest.NewRecorder()
-	ctx, _ := gin.CreateTestContext(w)
-	ctx.Request = httptest.NewRequest("GET", "/api/ready", nil)
-
-	apiReady(nil)(ctx)
-
-	c.Check(w.Code, Equals, 503)
-	c.Check(w.Body.String(), Matches, `.*"Status":"Aptly is unavailable".*`)
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 400)
 }
 
-func (s *FilesSuite) TestTruthy(c *C) {
-	// Test string values
-	c.Check(truthy("yes"), Equals, true)
-	c.Check(truthy("true"), Equals, true)
-	c.Check(truthy("1"), Equals, true)
-	c.Check(truthy("on"), Equals, true)
-	c.Check(truthy("anything"), Equals, true)
-	c.Check(truthy("n"), Equals, false)
-	c.Check(truthy("no"), Equals, false)
-	c.Check(truthy("f"), Equals, false)
-	c.Check(truthy("false"), Equals, false)
-	c.Check(truthy("0"), Equals, false)
-	c.Check(truthy("off"), Equals, false)
-	c.Check(truthy("NO"), Equals, false)    // case insensitive
-	c.Check(truthy("FALSE"), Equals, false) // case insensitive
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
+	_ = os.RemoveAll(s.aptlyContext.UploadPath())
+	c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
 
-	// Test int values
-	c.Check(truthy(1), Equals, true)
-	c.Check(truthy(42), Equals, true)
-	c.Check(truthy(-1), Equals, true)
-	c.Check(truthy(0), Equals, false)
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
 
-	// Test bool values
-	c.Check(truthy(true), Equals, true)
-	c.Check(truthy(false), Equals, false)
+	req, err := http.NewRequest("POST", "/api/files/testdir", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
 
-	// Test nil
-	c.Check(truthy(nil), Equals, false)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
+	// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
+	form := &multipart.Form{
+		File: map[string][]*multipart.FileHeader{
+			"file": {{Filename: "broken.bin"}},
+		},
+	}
+
+	req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
+	c.Assert(err, IsNil)
+	req.MultipartForm = form
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
+	base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.Chmod(base, 0555), IsNil)
+	defer func() { _ = os.Chmod(base, 0777) }()
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+	part, err := writer.CreateFormFile("file", "a.txt")
+	c.Assert(err, IsNil)
+	_, err = part.Write([]byte("x"))
+	c.Assert(err, IsNil)
+	c.Assert(writer.Close(), IsNil)
+
+	req, err := http.NewRequest("POST", "/api/files/readonly", body)
+	c.Assert(err, IsNil)
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
+}
+
+func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
+	parent := s.aptlyContext.UploadPath()
+	base := filepath.Join(parent, "cantremove")
+	c.Assert(os.MkdirAll(base, 0777), IsNil)
+	c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
+
+	c.Assert(os.Chmod(parent, 0555), IsNil)
+	defer func() { _ = os.Chmod(parent, 0777) }()
+
+	req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
+	c.Assert(err, IsNil)
+	w := httptest.NewRecorder()
+	s.router.ServeHTTP(w, req)
+	c.Assert(w.Code, Equals, 500)
 }

api/gpg.go

@@ -28,7 +28,7 @@ type gpgAddKeyParams struct {
 // @Summary Add GPG Keys
 // @Description **Adds GPG keys to aptly keyring**
 // @Description
-// @Description Add GPG public keys for veryfing remote repositories for mirroring.
+// @Description Add GPG public keys for verifying remote repositories for mirroring.
 // @Description
 // @Description Keys can be added in two ways:
 // @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)

api/gpg_test.go

@@ -1,213 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"net/http"
-	"net/http/httptest"
-
-	"github.com/gin-gonic/gin"
-	. "gopkg.in/check.v1"
-)
-
-type GPGTestSuite struct {
-	router *gin.Engine
-}
-
-var _ = Suite(&GPGTestSuite{})
-
-func (s *GPGTestSuite) SetUpTest(c *C) {
-	s.router = gin.New()
-	s.router.POST("/api/gpg/key", apiGPGAddKey)
-
-	gin.SetMode(gin.TestMode)
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyStructure(c *C) {
-	// Test GPG key add endpoint structure with sample key data
-	keyData := `-----BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFKuaIQBEAC+JC5od6Vw1tz2SEfBE7tBLQhNy3z2SIu7iNC3Bi/W6xUy5YKw
-sample key data for testing
------END PGP PUBLIC KEY BLOCK-----`
-
-	req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(keyData))
-	req.Header.Set("Content-Type", "application/pgp-keys")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will likely error due to no context or invalid key, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyEmptyBody(c *C) {
-	// Test GPG key add with empty body
-	req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(""))
-	req.Header.Set("Content-Type", "application/pgp-keys")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle empty body gracefully
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyInvalidData(c *C) {
-	// Test GPG key add with invalid key data
-	invalidKeys := []string{
-		"not a pgp key",
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\ninvalid\n-----END PGP PUBLIC KEY BLOCK-----",
-		"random text data",
-		"<xml>not a key</xml>",
-		"-----BEGIN CERTIFICATE-----\ninvalid cert\n-----END CERTIFICATE-----",
-	}
-
-	for _, keyData := range invalidKeys {
-		req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(keyData))
-		req.Header.Set("Content-Type", "application/pgp-keys")
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle invalid key data gracefully without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Key data: %s", keyData[:min(len(keyData), 50)]))
-	}
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyHTTPMethods(c *C) {
-	// Test that only POST method is allowed
-	deniedMethods := []string{"GET", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
-
-	for _, method := range deniedMethods {
-		req, _ := http.NewRequest(method, "/api/gpg/key", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 404, Commentf("Method: %s should be denied", method))
-	}
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyContentTypes(c *C) {
-	// Test different content types
-	contentTypes := []string{
-		"application/pgp-keys",
-		"text/plain",
-		"application/x-pgp-message",
-		"application/octet-stream",
-		"",
-	}
-
-	keyData := "-----BEGIN PGP PUBLIC KEY BLOCK-----\nsample\n-----END PGP PUBLIC KEY BLOCK-----"
-
-	for _, contentType := range contentTypes {
-		req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(keyData))
-		if contentType != "" {
-			req.Header.Set("Content-Type", contentType)
-		}
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle different content types without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Content-Type: %s", contentType))
-	}
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyLargePayload(c *C) {
-	// Test with large payload (simulate large key file)
-	largeKeyData := "-----BEGIN PGP PUBLIC KEY BLOCK-----\n"
-	for i := 0; i < 1000; i++ {
-		largeKeyData += "large key data line " + string(rune(i)) + "\n"
-	}
-	largeKeyData += "-----END PGP PUBLIC KEY BLOCK-----"
-
-	req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(largeKeyData))
-	req.Header.Set("Content-Type", "application/pgp-keys")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle large payloads without crashing
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyBinaryData(c *C) {
-	// Test with binary data
-	binaryData := []byte{0x00, 0x01, 0x02, 0x03, 0xFF, 0xFE, 0xFD}
-
-	req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBuffer(binaryData))
-	req.Header.Set("Content-Type", "application/octet-stream")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle binary data without crashing
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *GPGTestSuite) TestGPGAddKeySpecialCharacters(c *C) {
-	// Test with special characters and encoding
-	specialKeys := []string{
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\nключ с русскими символами\n-----END PGP PUBLIC KEY BLOCK-----",
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\n中文字符测试\n-----END PGP PUBLIC KEY BLOCK-----",
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\n🔑 emoji key 🔐\n-----END PGP PUBLIC KEY BLOCK-----",
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\n\"quotes\" and 'apostrophes'\n-----END PGP PUBLIC KEY BLOCK-----",
-		"-----BEGIN PGP PUBLIC KEY BLOCK-----\n<>&\"'`\n-----END PGP PUBLIC KEY BLOCK-----",
-	}
-
-	for i, keyData := range specialKeys {
-		req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(keyData))
-		req.Header.Set("Content-Type", "application/pgp-keys; charset=utf-8")
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle special characters without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Special key test #%d", i+1))
-	}
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyErrorHandling(c *C) {
-	// Test various error conditions
-	errorTests := []struct {
-		description string
-		data        string
-		contentType string
-		expectError bool
-	}{
-		{"Empty key", "", "application/pgp-keys", true},
-		{"Malformed header", "-----BEGIN WRONG BLOCK-----\ndata\n-----END WRONG BLOCK-----", "application/pgp-keys", true},
-		{"Missing end", "-----BEGIN PGP PUBLIC KEY BLOCK-----\ndata", "application/pgp-keys", true},
-		{"Missing begin", "data\n-----END PGP PUBLIC KEY BLOCK-----", "application/pgp-keys", true},
-		{"Only whitespace", "   \n\t\r\n   ", "application/pgp-keys", true},
-		{"JSON data", `{"key": "value"}`, "application/json", true},
-		{"XML data", `<key>value</key>`, "application/xml", true},
-	}
-
-	for _, test := range errorTests {
-		req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(test.data))
-		req.Header.Set("Content-Type", test.contentType)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle errors gracefully without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", test.description))
-	}
-}
-
-func (s *GPGTestSuite) TestGPGAddKeyReliability(c *C) {
-	// Test multiple sequential calls for reliability
-	keyData := "-----BEGIN PGP PUBLIC KEY BLOCK-----\ntest key data\n-----END PGP PUBLIC KEY BLOCK-----"
-
-	for i := 0; i < 5; i++ {
-		req, _ := http.NewRequest("POST", "/api/gpg/key", bytes.NewBufferString(keyData))
-		req.Header.Set("Content-Type", "application/pgp-keys")
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should be consistent across multiple calls
-		c.Check(w.Code, Not(Equals), 0, Commentf("Call #%d", i+1))
-	}
-}
-
-// Helper function for minimum of two integers
-func min(a, b int) int {
-	if a < b {
-		return a
-	}
-	return b
-}

api/graph_test.go

@@ -1,381 +0,0 @@
-package api
-
-import (
-	"mime"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-
-	. "gopkg.in/check.v1"
-)
-
-type GraphTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&GraphTestSuite{})
-
-func (s *GraphTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *GraphTestSuite) TestGraphDotFormat(c *C) {
-	// Test requesting raw DOT format
-	req, _ := http.NewRequest("GET", "/api/graph.dot", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should succeed with context and return DOT format
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "text/plain; charset=utf-8")
-}
-
-func (s *GraphTestSuite) TestGraphGvFormat(c *C) {
-	// Test requesting GV format (alias for DOT)
-	req, _ := http.NewRequest("GET", "/api/graph.gv", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should succeed with context and return DOT format (gv is alias)
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "text/plain; charset=utf-8")
-}
-
-func (s *GraphTestSuite) TestGraphSvgFormat(c *C) {
-	// Test requesting SVG format (requires graphviz)
-	req, _ := http.NewRequest("GET", "/api/graph.svg", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will likely error due to no context or missing graphviz
-	c.Check(w.Code, Not(Equals), 200) // Expect error
-}
-
-func (s *GraphTestSuite) TestGraphPngFormat(c *C) {
-	// Test requesting PNG format (requires graphviz)
-	req, _ := http.NewRequest("GET", "/api/graph.png", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will likely error due to no context or missing graphviz
-	c.Check(w.Code, Not(Equals), 200) // Expect error
-}
-
-func (s *GraphTestSuite) TestGraphWithHorizontalLayout(c *C) {
-	// Test with horizontal layout parameter
-	req, _ := http.NewRequest("GET", "/api/graph.svg?layout=horizontal", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will likely error due to no context, but should parse layout parameter
-	c.Check(w.Code, Not(Equals), 200) // Expect error due to missing context
-}
-
-func (s *GraphTestSuite) TestGraphWithVerticalLayout(c *C) {
-	// Test with vertical layout parameter
-	req, _ := http.NewRequest("GET", "/api/graph.png?layout=vertical", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will likely error due to no context, but should parse layout parameter
-	c.Check(w.Code, Not(Equals), 200) // Expect error due to missing context
-}
-
-func (s *GraphTestSuite) TestGraphWithInvalidLayout(c *C) {
-	// Test with invalid layout parameter
-	req, _ := http.NewRequest("GET", "/api/graph.dot?layout=invalid", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should succeed - invalid layout is ignored
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *GraphTestSuite) TestGraphWithEmptyLayout(c *C) {
-	// Test with empty layout parameter
-	req, _ := http.NewRequest("GET", "/api/graph.svg?layout=", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will fail because SVG requires graphviz which is not installed
-	c.Check(w.Code, Equals, 500)
-}
-
-func (s *GraphTestSuite) TestGraphWithMultipleParams(c *C) {
-	// Test with multiple query parameters
-	req, _ := http.NewRequest("GET", "/api/graph.png?layout=vertical&extra=param&another=value", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will fail because PNG requires graphviz which is not installed
-	c.Check(w.Code, Equals, 500)
-}
-
-func (s *GraphTestSuite) TestGraphParameterHandling(c *C) {
-	// Test parameter extraction and validation
-	testCases := []struct {
-		path        string
-		description string
-	}{
-		{"/api/graph.dot", "DOT format"},
-		{"/api/graph.gv", "GV format"},
-		{"/api/graph.svg", "SVG format"},
-		{"/api/graph.png", "PNG format"},
-		{"/api/graph.pdf", "PDF format"},
-		{"/api/graph.ps", "PostScript format"},
-		{"/api/graph.jpg", "JPEG format"},
-		{"/api/graph.gif", "GIF format"},
-		{"/api/graph.unknown", "Unknown format"},
-	}
-
-	for _, tc := range testCases {
-		req, _ := http.NewRequest("GET", tc.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// All should return some response without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test case: %s", tc.description))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphMimeTypeHandling(c *C) {
-	// Test MIME type detection for different extensions
-	extensions := map[string]string{
-		"svg": "image/svg+xml",
-		"png": "image/png",
-		"pdf": "application/pdf",
-		"ps":  "application/postscript",
-		"jpg": "image/jpeg",
-		"gif": "image/gif",
-	}
-
-	for ext, expectedMime := range extensions {
-		actualMime := mime.TypeByExtension("." + ext)
-		if actualMime != "" {
-			// Just check that the actual MIME type starts with expected
-			c.Check(strings.HasPrefix(actualMime, expectedMime), Equals, true,
-				Commentf("MIME type mismatch for extension: %s", ext))
-		}
-	}
-}
-
-func (s *GraphTestSuite) TestGraphHTTPMethods(c *C) {
-	// Test that only GET method is allowed
-	deniedMethods := []string{"POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
-
-	for _, method := range deniedMethods {
-		req, _ := http.NewRequest(method, "/api/graph.svg", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 404, Commentf("Method: %s should be denied", method))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphPathValidation(c *C) {
-	// Test path validation and parameter extraction
-	validPaths := []string{
-		"/api/graph.dot",
-		"/api/graph.svg",
-		"/api/graph.png",
-		"/api/graph.pdf",
-	}
-
-	invalidPaths := []string{
-		"/api/graph",           // Missing extension
-		"/api/graph.",          // Empty extension
-		"/api/graphs.svg",      // Wrong endpoint name
-	}
-
-	for _, path := range validPaths {
-		req, _ := http.NewRequest("GET", path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should match route (even if it errors due to missing context)
-		c.Check(w.Code, Not(Equals), 404, Commentf("Valid path should match route: %s", path))
-	}
-
-	for _, path := range invalidPaths {
-		req, _ := http.NewRequest("GET", path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not match route
-		c.Check(w.Code, Equals, 404, Commentf("Invalid path should not match route: %s", path))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphExtensionExtraction(c *C) {
-	// Test that extension is properly extracted from path
-	testPaths := []string{
-		"/api/graph.dot",
-		"/api/graph.svg",
-		"/api/graph.png",
-		"/api/graph.pdf",
-		"/api/graph.ps",
-		"/api/graph.jpg",
-		"/api/graph.gif",
-		"/api/graph.unknown",
-	}
-
-	for _, path := range testPaths {
-		req, _ := http.NewRequest("GET", path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle extension extraction without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Extension extraction failed for: %s", path))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphQueryParameterHandling(c *C) {
-	// Test various query parameter combinations
-	queryTests := []struct {
-		query       string
-		description string
-	}{
-		{"", "no parameters"},
-		{"layout=horizontal", "horizontal layout"},
-		{"layout=vertical", "vertical layout"},
-		{"layout=invalid", "invalid layout"},
-		{"layout=", "empty layout"},
-		{"layout=horizontal&extra=param", "multiple parameters"},
-		{"unknown=param", "unknown parameter"},
-		{"layout=horizontal&layout=vertical", "duplicate parameters"},
-	}
-
-	for _, test := range queryTests {
-		path := "/api/graph.svg"
-		if test.query != "" {
-			path += "?" + test.query
-		}
-
-		req, _ := http.NewRequest("GET", path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle query parameters without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Query parameter test: %s", test.description))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphErrorHandling(c *C) {
-	// Test various error conditions
-	errorTests := []struct {
-		path        string
-		description string
-	}{
-		{"/api/graph.svg", "missing database context"},
-		{"/api/graph.png", "missing graphviz"},
-		{"/api/graph.unknown", "unknown format"},
-		{"/api/graph.dot", "raw DOT format"},
-	}
-
-	for _, test := range errorTests {
-		req, _ := http.NewRequest("GET", test.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle errors gracefully without panicking
-		c.Check(w.Code, Not(Equals), 0, Commentf("Error test: %s", test.description))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphContentTypeHeaders(c *C) {
-	// Test that appropriate content types are set for different formats
-	formatTests := []struct {
-		ext         string
-		expectJSON  bool
-		expectImage bool
-	}{
-		{"dot", false, false}, // Should return text
-		{"gv", false, false},  // Should return text
-		{"svg", false, true},  // Should return image/svg+xml (if successful)
-		{"png", false, true},  // Should return image/png (if successful)
-		{"pdf", false, false}, // Should return application/pdf (if successful)
-	}
-
-	for _, test := range formatTests {
-		req, _ := http.NewRequest("GET", "/api/graph."+test.ext, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		contentType := w.Header().Get("Content-Type")
-
-		if test.expectJSON {
-			c.Check(strings.Contains(contentType, "application/json"), Equals, true,
-				Commentf("Expected JSON content type for .%s, got: %s", test.ext, contentType))
-		}
-
-		// Note: Image content types will only be set if graphviz is available and context exists
-		c.Check(contentType, Not(Equals), "", Commentf("Content type should be set for .%s", test.ext))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphSpecialCharacters(c *C) {
-	// Test handling of special characters in query parameters
-	specialQueries := []string{
-		"layout=horizontal%20with%20spaces",
-		"layout=vertical&param=value%20with%20spaces",
-		"layout=test%26special%3Dchars",
-		"layout=unicode%E2%9C%93",
-		"param=%3Cscript%3Ealert%28%29%3C%2Fscript%3E", // XSS attempt
-	}
-
-	for _, query := range specialQueries {
-		req, _ := http.NewRequest("GET", "/api/graph.svg?"+query, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle special characters without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Special character test failed for: %s", query))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphLargeExtensions(c *C) {
-	// Test with very long extensions
-	longExt := strings.Repeat("x", 1000)
-	req, _ := http.NewRequest("GET", "/api/graph."+longExt, nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle long extensions without crashing
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *GraphTestSuite) TestGraphReliability(c *C) {
-	// Test multiple sequential calls for reliability
-	for i := 0; i < 5; i++ {
-		req, _ := http.NewRequest("GET", "/api/graph.dot", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should be consistent across multiple calls
-		c.Check(w.Code, Not(Equals), 0, Commentf("Call #%d", i+1))
-	}
-}
-
-func (s *GraphTestSuite) TestGraphConcurrency(c *C) {
-	// Test concurrent requests to ensure thread safety
-	done := make(chan bool, 5)
-
-	for i := 0; i < 5; i++ {
-		go func(id int) {
-			defer func() { done <- true }()
-
-			req, _ := http.NewRequest("GET", "/api/graph.svg", nil)
-			w := httptest.NewRecorder()
-			s.router.ServeHTTP(w, req)
-
-			// Should handle concurrent requests without issues
-		}(i)
-	}
-
-	// Wait for all requests to complete
-	for i := 0; i < 5; i++ {
-		<-done
-	}
-
-	c.Check(true, Equals, true) // Test completed without deadlocks
-}

api/metrics_test.go

@@ -1,600 +0,0 @@
-package api
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"runtime"
-	"strings"
-
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/gin-gonic/gin"
-	"github.com/prometheus/client_golang/prometheus"
-	dto "github.com/prometheus/client_model/go"
-	. "gopkg.in/check.v1"
-)
-
-type MetricsTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&MetricsTestSuite{})
-
-func (s *MetricsTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-	// Reset metrics registrar state for each test
-	MetricsCollectorRegistrar.hasRegistered = false
-}
-
-func (s *MetricsTestSuite) TestMetricsCollectorRegistrarRegisterOnce(c *C) {
-	// Test that metrics are only registered once
-	registrar := &metricsCollectorRegistrar{hasRegistered: false}
-
-	// First registration should work
-	registrar.Register(s.router.(*gin.Engine))
-	c.Check(registrar.hasRegistered, Equals, true)
-
-	// Second registration should be skipped
-	registrar.Register(s.router.(*gin.Engine))
-	c.Check(registrar.hasRegistered, Equals, true)
-}
-
-func (s *MetricsTestSuite) TestMetricsCollectorRegistrarVersionGauge(c *C) {
-	// Test that version gauge is set correctly
-	registrar := &metricsCollectorRegistrar{hasRegistered: false}
-
-	// Register metrics
-	registrar.Register(s.router.(*gin.Engine))
-
-	// Check that version gauge was set
-	expectedLabels := prometheus.Labels{
-		"version":   aptly.Version,
-		"goversion": runtime.Version(),
-	}
-
-	gauge := apiVersionGauge.With(expectedLabels)
-	c.Check(gauge, NotNil)
-
-	// Verify the gauge value is 1
-	metric := &dto.Metric{}
-	gauge.(prometheus.Gauge).Write(metric)
-	c.Check(metric.GetGauge().GetValue(), Equals, float64(1))
-}
-
-func (s *MetricsTestSuite) TestApiRequestsInFlightGauge(c *C) {
-	// Test that in-flight requests gauge works
-	c.Check(apiRequestsInFlightGauge, NotNil)
-
-	// Test that we can create labels for the gauge
-	gauge := apiRequestsInFlightGauge.WithLabelValues("GET", "/api/test")
-	c.Check(gauge, NotNil)
-
-	// Test incrementing and decrementing
-	gauge.Inc()
-	gauge.Dec()
-}
-
-func (s *MetricsTestSuite) TestApiRequestsTotalCounter(c *C) {
-	// Test that total requests counter works
-	c.Check(apiRequestsTotalCounter, NotNil)
-
-	// Test that we can create labels for the counter
-	counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", "/api/test")
-	c.Check(counter, NotNil)
-
-	// Test incrementing
-	counter.Inc()
-}
-
-func (s *MetricsTestSuite) TestApiRequestSizeSummary(c *C) {
-	// Test that request size summary works
-	c.Check(apiRequestSizeSummary, NotNil)
-
-	// Test that we can create labels for the summary
-	summary := apiRequestSizeSummary.WithLabelValues("200", "POST", "/api/test")
-	c.Check(summary, NotNil)
-
-	// Test observing values
-	summary.Observe(1024.0)
-	summary.Observe(512.0)
-}
-
-func (s *MetricsTestSuite) TestApiResponseSizeSummary(c *C) {
-	// Test that response size summary works
-	c.Check(apiResponseSizeSummary, NotNil)
-
-	// Test that we can create labels for the summary
-	summary := apiResponseSizeSummary.WithLabelValues("200", "GET", "/api/test")
-	c.Check(summary, NotNil)
-
-	// Test observing values
-	summary.Observe(2048.0)
-	summary.Observe(1024.0)
-}
-
-func (s *MetricsTestSuite) TestApiRequestsDurationSummary(c *C) {
-	// Test that request duration summary works
-	c.Check(apiRequestsDurationSummary, NotNil)
-
-	// Test that we can create labels for the summary
-	summary := apiRequestsDurationSummary.WithLabelValues("200", "GET", "/api/test")
-	c.Check(summary, NotNil)
-
-	// Test observing duration values
-	summary.Observe(0.1)  // 100ms
-	summary.Observe(0.05) // 50ms
-	summary.Observe(1.0)  // 1s
-}
-
-func (s *MetricsTestSuite) TestApiFilesUploadedCounter(c *C) {
-	// Test that files uploaded counter works
-	c.Check(apiFilesUploadedCounter, NotNil)
-
-	// Test that we can create labels for the counter
-	counter := apiFilesUploadedCounter.WithLabelValues("uploads")
-	c.Check(counter, NotNil)
-
-	// Test incrementing
-	counter.Inc()
-	counter.Add(5)
-}
-
-func (s *MetricsTestSuite) TestApiReposPackageCountGauge(c *C) {
-	// Test that repos package count gauge works
-	c.Check(apiReposPackageCountGauge, NotNil)
-
-	// Test that we can create labels for the gauge
-	gauge := apiReposPackageCountGauge.WithLabelValues("source", "stable", "main")
-	c.Check(gauge, NotNil)
-
-	// Test setting values
-	gauge.Set(100)
-	gauge.Set(150)
-	gauge.Inc()
-	gauge.Dec()
-}
-
-func (s *MetricsTestSuite) TestMetricsPrometheusIntegration(c *C) {
-	// Test integration with Prometheus client library
-
-	// Test that metrics are properly registered with default registry
-	metricNames := []string{
-		"aptly_api_http_requests_in_flight",
-		"aptly_api_http_requests_total",
-		"aptly_api_http_request_size_bytes",
-		"aptly_api_http_response_size_bytes",
-		"aptly_api_http_request_duration_seconds",
-		"aptly_build_info",
-		"aptly_api_files_uploaded_total",
-		"aptly_repos_package_count",
-	}
-
-	for _, metricName := range metricNames {
-		// Try to gather metrics to ensure they're registered
-		gathered, err := prometheus.DefaultGatherer.Gather()
-		c.Check(err, IsNil)
-
-		found := false
-		for _, metricFamily := range gathered {
-			if metricFamily.GetName() == metricName {
-				found = true
-				break
-			}
-		}
-		c.Check(found, Equals, true, Commentf("Metric %s not found", metricName))
-	}
-}
-
-func (s *MetricsTestSuite) TestMetricsLabels(c *C) {
-	// Test that metrics have expected labels
-
-	// Test in-flight gauge labels
-	gauge := apiRequestsInFlightGauge.WithLabelValues("GET", "/api/test")
-	c.Check(gauge, NotNil)
-
-	// Test total counter labels
-	counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", "/api/test")
-	c.Check(counter, NotNil)
-
-	// Test request size summary labels
-	requestSummary := apiRequestSizeSummary.WithLabelValues("200", "POST", "/api/upload")
-	c.Check(requestSummary, NotNil)
-
-	// Test response size summary labels
-	responseSummary := apiResponseSizeSummary.WithLabelValues("404", "GET", "/api/missing")
-	c.Check(responseSummary, NotNil)
-
-	// Test duration summary labels
-	durationSummary := apiRequestsDurationSummary.WithLabelValues("500", "POST", "/api/error")
-	c.Check(durationSummary, NotNil)
-
-	// Test version gauge labels
-	versionGauge := apiVersionGauge.WithLabelValues("1.0.0", "go1.19")
-	c.Check(versionGauge, NotNil)
-
-	// Test files uploaded counter labels
-	filesCounter := apiFilesUploadedCounter.WithLabelValues("temp-uploads")
-	c.Check(filesCounter, NotNil)
-
-	// Test repos package count gauge labels
-	reposGauge := apiReposPackageCountGauge.WithLabelValues("snapshot:test", "testing", "contrib")
-	c.Check(reposGauge, NotNil)
-}
-
-func (s *MetricsTestSuite) TestMetricsWithDifferentHTTPCodes(c *C) {
-	// Test metrics with various HTTP status codes
-	httpCodes := []string{"200", "201", "400", "401", "403", "404", "409", "500", "502", "503"}
-
-	for _, code := range httpCodes {
-		// Test that metrics work with different status codes
-		counter := apiRequestsTotalCounter.WithLabelValues(code, "GET", "/api/test")
-		counter.Inc()
-
-		requestSummary := apiRequestSizeSummary.WithLabelValues(code, "POST", "/api/test")
-		requestSummary.Observe(100)
-
-		responseSummary := apiResponseSizeSummary.WithLabelValues(code, "GET", "/api/test")
-		responseSummary.Observe(200)
-
-		durationSummary := apiRequestsDurationSummary.WithLabelValues(code, "PUT", "/api/test")
-		durationSummary.Observe(0.1)
-	}
-}
-
-func (s *MetricsTestSuite) TestMetricsWithDifferentHTTPMethods(c *C) {
-	// Test metrics with various HTTP methods
-	httpMethods := []string{"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
-
-	for _, method := range httpMethods {
-		// Test that metrics work with different HTTP methods
-		gauge := apiRequestsInFlightGauge.WithLabelValues(method, "/api/test")
-		gauge.Inc()
-		gauge.Dec()
-
-		counter := apiRequestsTotalCounter.WithLabelValues("200", method, "/api/test")
-		counter.Inc()
-	}
-}
-
-func (s *MetricsTestSuite) TestMetricsWithDifferentPaths(c *C) {
-	// Test metrics with various API paths
-	apiPaths := []string{
-		"/api/repos",
-		"/api/repos/test",
-		"/api/snapshots",
-		"/api/publish",
-		"/api/files",
-		"/api/files/upload",
-		"/api/mirrors",
-		"/api/tasks",
-		"/api/version",
-	}
-
-	for _, path := range apiPaths {
-		counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", path)
-		counter.Inc()
-
-		gauge := apiRequestsInFlightGauge.WithLabelValues("GET", path)
-		gauge.Inc()
-		gauge.Dec()
-	}
-}
-
-func (s *MetricsTestSuite) TestMetricsThreadSafety(c *C) {
-	// Test that metrics are thread-safe by simulating concurrent access
-	done := make(chan bool, 10)
-
-	for i := 0; i < 10; i++ {
-		go func(id int) {
-			defer func() { done <- true }()
-
-			// Simulate concurrent metric updates
-			for j := 0; j < 100; j++ {
-				counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", "/api/concurrent")
-				counter.Inc()
-
-				gauge := apiRequestsInFlightGauge.WithLabelValues("GET", "/api/concurrent")
-				gauge.Inc()
-				gauge.Dec()
-
-				summary := apiRequestsDurationSummary.WithLabelValues("200", "GET", "/api/concurrent")
-				summary.Observe(0.01)
-			}
-		}(i)
-	}
-
-	// Wait for all goroutines to complete
-	for i := 0; i < 10; i++ {
-		<-done
-	}
-
-	// Verify metrics were updated (exact count doesn't matter due to concurrency)
-	c.Check(true, Equals, true) // Test completed without race conditions
-}
-
-func (s *MetricsTestSuite) TestMetricsMetadata(c *C) {
-	// Test that metrics have proper metadata (help text, names)
-
-	// Gather all metrics
-	gathered, err := prometheus.DefaultGatherer.Gather()
-	c.Check(err, IsNil)
-
-	expectedMetrics := map[string]string{
-		"aptly_api_http_requests_in_flight":       "Number of concurrent HTTP api requests currently handled.",
-		"aptly_api_http_requests_total":           "Total number of api requests.",
-		"aptly_api_http_request_size_bytes":       "Api HTTP request size in bytes.",
-		"aptly_api_http_response_size_bytes":      "Api HTTP response size in bytes.",
-		"aptly_api_http_request_duration_seconds": "Duration of api requests in seconds.",
-		"aptly_build_info":                        "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
-		"aptly_api_files_uploaded_total":          "Total number of uploaded files labeled by upload directory.",
-		"aptly_repos_package_count":               "Current number of published packages labeled by source, distribution and component.",
-	}
-
-	for _, metricFamily := range gathered {
-		metricName := metricFamily.GetName()
-		if expectedHelp, exists := expectedMetrics[metricName]; exists {
-			c.Check(metricFamily.GetHelp(), Equals, expectedHelp,
-				Commentf("Help text mismatch for metric: %s", metricName))
-		}
-	}
-}
-
-func (s *MetricsTestSuite) TestCountPackagesByRepos(c *C) {
-	// Test countPackagesByRepos function structure
-	// Note: This function requires database context which we don't have in tests,
-	// but we can test that it doesn't crash when called
-
-	// This will likely error due to no context, but should not panic
-	defer func() {
-		if r := recover(); r != nil {
-			c.Fatalf("countPackagesByRepos panicked: %v", r)
-		}
-	}()
-
-	countPackagesByRepos()
-
-	// If we get here, the function didn't panic
-	c.Check(true, Equals, true)
-}
-
-func (s *MetricsTestSuite) TestGetBasePath(c *C) {
-	// Test getBasePath function
-	w := httptest.NewRecorder()
-	ginCtx, _ := gin.CreateTestContext(w)
-	
-	// Test with simple path (only returns first two segments)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/version", nil)
-	basePath := getBasePath(ginCtx)
-	c.Check(basePath, Equals, "/api/version")
-	
-	// Test with path containing more segments (still returns first two)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/repos/test-repo", nil)
-	basePath = getBasePath(ginCtx)
-	c.Check(basePath, Equals, "/api/repos")
-	
-	// Test with nested parameters (still returns first two)
-	ginCtx.Request = httptest.NewRequest("GET", "/api/repos/repo1/packages", nil)
-	basePath = getBasePath(ginCtx)
-	c.Check(basePath, Equals, "/api/repos")
-	
-	// Test with root path
-	ginCtx.Request = httptest.NewRequest("GET", "/", nil)
-	basePath = getBasePath(ginCtx)
-	c.Check(basePath, Equals, "/")
-	
-	// Test with single segment
-	ginCtx.Request = httptest.NewRequest("GET", "/api", nil)
-	basePath = getBasePath(ginCtx)
-	c.Check(basePath, Equals, "/api")
-}
-
-func (s *MetricsTestSuite) TestGetURLSegment(c *C) {
-	// Test getURLSegment function
-	
-	// Test valid segments
-	segment, err := getURLSegment("/api/repos/test", 0)
-	c.Check(err, IsNil)
-	c.Check(*segment, Equals, "/api")
-	
-	segment, err = getURLSegment("/api/repos/test", 1)
-	c.Check(err, IsNil)
-	c.Check(*segment, Equals, "/repos")
-	
-	segment, err = getURLSegment("/api/repos/test", 2)
-	c.Check(err, IsNil)
-	c.Check(*segment, Equals, "/test")
-	
-	// Test out of range
-	_, err = getURLSegment("/api/repos", 3)
-	c.Check(err, NotNil)
-	
-	// Test root path
-	segment, err = getURLSegment("/", 0)
-	c.Check(err, NotNil) // No segments after removing empty string
-}
-
-func (s *MetricsTestSuite) TestInstrumentHandlerInFlight(c *C) {
-	// Test instrumentHandlerInFlight middleware
-	w := httptest.NewRecorder()
-	
-	// Create test gin context
-	router := gin.New()
-	
-	// Add instrumentation middleware
-	router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-	
-	// Add test handler
-	router.GET("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"status": "ok"})
-	})
-	
-	// Make request
-	req := httptest.NewRequest("GET", "/api/test", nil)
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *MetricsTestSuite) TestInstrumentHandlerCounter(c *C) {
-	// Test instrumentHandlerCounter middleware
-	w := httptest.NewRecorder()
-	
-	// Create test gin context
-	router := gin.New()
-	
-	// Add instrumentation middleware
-	router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-	
-	// Add test handler
-	router.GET("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"status": "ok"})
-	})
-	
-	// Make request
-	req := httptest.NewRequest("GET", "/api/test", nil)
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *MetricsTestSuite) TestInstrumentHandlerRequestSize(c *C) {
-	// Test instrumentHandlerRequestSize middleware
-	w := httptest.NewRecorder()
-	
-	// Create test gin context
-	router := gin.New()
-	
-	// Add instrumentation middleware
-	router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-	
-	// Add test handler
-	router.POST("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"status": "ok"})
-	})
-	
-	// Make request with body
-	req := httptest.NewRequest("POST", "/api/test", strings.NewReader("test body"))
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *MetricsTestSuite) TestInstrumentHandlerResponseSize(c *C) {
-	// Test instrumentHandlerResponseSize middleware
-	w := httptest.NewRecorder()
-	
-	// Create test gin context
-	router := gin.New()
-	
-	// Add instrumentation middleware
-	router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-	
-	// Add test handler
-	router.GET("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"data": strings.Repeat("x", 1000)})
-	})
-	
-	// Make request
-	req := httptest.NewRequest("GET", "/api/test", nil)
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *MetricsTestSuite) TestInstrumentHandlerDuration(c *C) {
-	// Test instrumentHandlerDuration middleware
-	w := httptest.NewRecorder()
-	
-	// Create test gin context
-	router := gin.New()
-	
-	// Add instrumentation middleware
-	router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
-	
-	// Add test handler
-	router.GET("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"status": "ok"})
-	})
-	
-	// Make request
-	req := httptest.NewRequest("GET", "/api/test", nil)
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *MetricsTestSuite) TestMetricsRegistration(c *C) {
-	// Test that metrics registration works correctly with gin router
-	MetricsCollectorRegistrar.Register(s.router.(*gin.Engine))
-
-	// Create a test request to trigger middleware
-	req, _ := http.NewRequest("GET", "/test", nil)
-	w := httptest.NewRecorder()
-
-	// Add a test handler
-	s.router.(*gin.Engine).GET("/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"test": "response"})
-	})
-
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(MetricsCollectorRegistrar.hasRegistered, Equals, true)
-}
-
-func (s *MetricsTestSuite) TestMetricsErrorConditions(c *C) {
-	// Test error handling in metrics collection
-
-	// Test with invalid label values (should not crash)
-	invalidLabels := []string{"", "very_long_label_" + strings.Repeat("x", 1000), "label\nwith\nnewlines"}
-
-	for _, label := range invalidLabels {
-		// These should not crash, even with invalid labels
-		gauge := apiRequestsInFlightGauge.WithLabelValues("GET", label)
-		gauge.Inc()
-		gauge.Dec()
-
-		counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", label)
-		counter.Inc()
-	}
-}
-
-func (s *MetricsTestSuite) TestMetricsValueRanges(c *C) {
-	// Test metrics with various value ranges
-
-	// Test large values
-	summary := apiRequestSizeSummary.WithLabelValues("200", "POST", "/api/large")
-	summary.Observe(1e9)  // 1GB
-	summary.Observe(1e12) // 1TB
-
-	// Test very small values
-	durationSummary := apiRequestsDurationSummary.WithLabelValues("200", "GET", "/api/fast")
-	durationSummary.Observe(1e-9) // 1 nanosecond
-	durationSummary.Observe(1e-6) // 1 microsecond
-
-	// Test zero values
-	gauge := apiReposPackageCountGauge.WithLabelValues("empty", "dist", "comp")
-	gauge.Set(0)
-
-	// Test negative values (should be handled gracefully)
-	gauge.Set(-1) // May or may not be allowed by Prometheus, but shouldn't crash
-}
-
-func (s *MetricsTestSuite) TestMetricsWithSpecialCharacters(c *C) {
-	// Test metrics with special characters in labels
-	specialPaths := []string{
-		"/api/repos/repo-with-dashes",
-		"/api/repos/repo_with_underscores",
-		"/api/repos/repo.with.dots",
-		"/api/repos/repo+with+plus",
-		"/api/repos/repo%20with%20encoded",
-	}
-
-	for _, path := range specialPaths {
-		counter := apiRequestsTotalCounter.WithLabelValues("200", "GET", path)
-		counter.Inc()
-
-		gauge := apiRequestsInFlightGauge.WithLabelValues("GET", path)
-		gauge.Inc()
-		gauge.Dec()
-	}
-}

api/middleware_test.go

@@ -253,28 +253,3 @@ func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
 	}
 	c.Check(*segment, Equals, "/repos")
 }
-
-func (s *MiddlewareSuite) TestInstrumentationMiddleware(c *C) {
-	// Test instrumentation middleware functions
-	router := gin.New()
-	
-	// Add all instrumentation middleware
-	router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
-	router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
-	router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
-	router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
-	router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
-	
-	// Add test handler
-	router.GET("/api/test", func(c *gin.Context) {
-		c.JSON(200, gin.H{"status": "ok"})
-	})
-	
-	// Make request
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest("GET", "/api/test", nil)
-	req.ContentLength = 42
-	router.ServeHTTP(w, req)
-	
-	c.Check(w.Code, Equals, 200)
-}

api/mirror.go

@@ -175,9 +175,9 @@ func apiMirrorsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	mirrorCollection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 
 	repo, err := mirrorCollection.ByName(name)
 	if err != nil {
@@ -187,21 +187,34 @@ func apiMirrorsDrop(c *gin.Context) {
 
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete mirror %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByRemoteRepoSource(repo)
+			// Fresh checks with current collections
+			snapshots := taskSnapshotCollection.ByRemoteRepoSource(repo)
 
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
 			}
 		}
 
-		err = mirrorCollection.Drop(repo)
+		err = taskMirrorCollection.Drop(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
 		}
@@ -232,6 +245,7 @@ func apiMirrorsShow(c *gin.Context) {
 	err = collection.LoadComplete(repo)
 	if err != nil {
 		AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
+		return
 	}
 
 	c.JSON(200, repo)
@@ -333,26 +347,8 @@ func apiMirrorsPackages(c *gin.Context) {
 type mirrorUpdateParams struct {
 	// Change mirror name to `Name`
 	Name string `                 json:"Name"                   example:"mirror1"`
-	// Url of the archive to mirror
-	ArchiveURL string `           json:"ArchiveURL"             example:"http://deb.debian.org/debian"`
-	// Package query that is applied to mirror packages
-	Filter string `               json:"Filter"                 example:"xserver-xorg"`
-	// Limit mirror to those architectures, if not specified aptly would fetch all architectures
-	Architectures []string `      json:"Architectures"          example:"amd64"`
-	// Components to mirror, if not specified aptly would fetch all components
-	Components []string `         json:"Components"             example:"main"`
-	// Gpg keyring(s) for verifing Release file
+	// Gpg keyring(s) for verifying Release file
 	Keyrings []string `           json:"Keyrings"               example:"trustedkeys.gpg"`
-	// Set "true" to include dependencies of matching packages when filtering
-	FilterWithDeps bool `         json:"FilterWithDeps"`
-	// Set "true" to mirror source packages
-	DownloadSources bool `        json:"DownloadSources"`
-	// Set "true" to mirror udeb files
-	DownloadUdebs bool `          json:"DownloadUdebs"`
-	// Set "true" to skip checking if the given components are in the Release file
-	SkipComponentCheck bool `     json:"SkipComponentCheck"`
-	// Set "true" to skip checking if the given architectures are in the Release file
-	SkipArchitectureCheck bool `  json:"SkipArchitectureCheck"`
 	// Set "true" to ignore checksum errors
 	IgnoreChecksums bool `        json:"IgnoreChecksums"`
 	// Set "true" to skip the verification of Release file signatures
@@ -387,21 +383,14 @@ func apiMirrorsUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.RemoteRepoCollection()
 
-	remote, err = collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	remote, err = collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	b.Name = remote.Name
-	b.DownloadUdebs = remote.DownloadUdebs
-	b.DownloadSources = remote.DownloadSources
-	b.SkipComponentCheck = remote.SkipComponentCheck
-	b.SkipArchitectureCheck = remote.SkipArchitectureCheck
-	b.FilterWithDeps = remote.FilterWithDeps
-	b.Filter = remote.Filter
-	b.Architectures = remote.Architectures
-	b.Components = remote.Components
 	b.IgnoreSignatures = context.Config().GpgDisableVerify
 
 	log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -410,6 +399,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Pre-task validation of new name if provided
 	if b.Name != remote.Name {
 		_, err = collection.ByName(b.Name)
 		if err == nil {
@@ -418,27 +408,6 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 	}
 
-	if b.DownloadUdebs != remote.DownloadUdebs {
-		if remote.IsFlat() && b.DownloadUdebs {
-			AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
-			return
-		}
-	}
-
-	if b.ArchiveURL != "" {
-		remote.SetArchiveRoot(b.ArchiveURL)
-	}
-
-	remote.Name = b.Name
-	remote.DownloadUdebs = b.DownloadUdebs
-	remote.DownloadSources = b.DownloadSources
-	remote.SkipComponentCheck = b.SkipComponentCheck
-	remote.SkipArchitectureCheck = b.SkipArchitectureCheck
-	remote.FilterWithDeps = b.FilterWithDeps
-	remote.Filter = b.Filter
-	remote.Architectures = b.Architectures
-	remote.Components = b.Components
-
 	verifier, err := getVerifier(b.Keyrings)
 	if err != nil {
 		AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -447,9 +416,26 @@ func apiMirrorsUpdate(c *gin.Context) {
 
 	resources := []string{string(remote.Key())}
 	maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.RemoteRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		remote, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Fresh rename check inside lock (if renaming)
+		if b.Name != remote.Name {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: mirror %s already exists", b.Name)
+			}
+		}
 
 		downloader := context.NewDownloader(out)
-		err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
+		err = remote.Fetch(downloader, verifier, b.IgnoreSignatures)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -461,7 +447,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
+		err = remote.DownloadPackageIndexes(out, downloader, verifier, taskCollectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -480,8 +466,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 			}
 		}
 
-		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
-			collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
+		queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(),
+			taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -491,12 +477,12 @@ func apiMirrorsUpdate(c *gin.Context) {
 			e := context.ReOpenDatabase()
 			if e == nil {
 				remote.MarkAsIdle()
-				_ = collection.Update(remote)
+				_ = taskCollection.Update(remote)
 			}
 		}()
 
 		remote.MarkAsUpdating()
-		err = collection.Update(remote)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
@@ -600,7 +586,7 @@ func apiMirrorsUpdate(c *gin.Context) {
 						}
 
 						// and import it back to the pool
-						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
+						task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, taskCollectionFactory.ChecksumCollection(nil))
 						if err != nil {
 							//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
 							pushError(err)
@@ -653,8 +639,8 @@ func apiMirrorsUpdate(c *gin.Context) {
 		}
 
 		log.Info().Msgf("%s: Finalizing download...", b.Name)
-		_ = remote.FinalizeDownload(collectionFactory, out)
-		err = collectionFactory.RemoteRepoCollection().Update(remote)
+		_ = remote.FinalizeDownload(taskCollectionFactory, out)
+		err = taskCollection.Update(remote)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}

api/mirror_test.go

@@ -38,30 +38,3 @@ func (s *MirrorSuite) TestCreateMirror(c *C) {
 	c.Check(response.Code, Equals, 400)
 	c.Check(response.Body.String(), Equals, "")
 }
-
-func (s *MirrorSuite) TestMirrorShow(c *C) {
-	// Test showing a specific mirror
-	response, _ := s.HTTPRequest("GET", "/api/mirrors/test-mirror", nil)
-	c.Check(response.Code, Equals, 404)
-}
-
-func (s *MirrorSuite) TestMirrorUpdate(c *C) {
-	// Test updating a mirror
-	body, _ := json.Marshal(gin.H{
-		"ArchiveURL": "http://new.archive.url/debian",
-	})
-	response, _ := s.HTTPRequest("PUT", "/api/mirrors/test-mirror", bytes.NewReader(body))
-	c.Check(response.Code, Equals, 404)
-}
-
-func (s *MirrorSuite) TestMirrorPackages(c *C) {
-	// Test listing packages in a mirror
-	response, _ := s.HTTPRequest("GET", "/api/mirrors/test-mirror/packages", nil)
-	c.Check(response.Code, Equals, 404)
-}
-
-func (s *MirrorSuite) TestMirrorUpdateRun(c *C) {
-	// Test running mirror update
-	response, _ := s.HTTPRequest("PUT", "/api/mirrors/test-mirror/update", nil)
-	c.Check(response.Code, Equals, 404)
-}

api/packages_test.go

@@ -1,10 +1,6 @@
 package api
 
 import (
-	"bytes"
-	"encoding/json"
-	
-	"github.com/gin-gonic/gin"
 	. "gopkg.in/check.v1"
 )
 
@@ -14,39 +10,9 @@ type PackagesSuite struct {
 
 var _ = Suite(&PackagesSuite{})
 
-func (s *PackagesSuite) TestPackageShow(c *C) {
-	// Test showing a specific package
-	response, _ := s.HTTPRequest("GET", "/api/packages/Pamd64%20test%201.0%20abc123", nil)
-	// Will return 404 as the package doesn't exist
-	c.Check(response.Code, Equals, 404)
-}
-
-func (s *PackagesSuite) TestPackagesList(c *C) {
-	// Test listing all packages
-	response, _ := s.HTTPRequest("GET", "/api/packages", nil)
-	c.Check(response.Code, Equals, 200)
-	
-	var result []interface{}
-	err := json.Unmarshal(response.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(result, NotNil)
-}
-
 func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
-	// Create dummy repo first
-	body, _ := json.Marshal(gin.H{"Name": "dummy"})
-	resp, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
-	c.Assert(err, IsNil)
-	c.Check(resp.Code, Equals, 201)
-	
-	// Now test packages with maximumVersion
 	response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
 	c.Assert(err, IsNil)
 	c.Check(response.Code, Equals, 200)
 	c.Check(response.Body.String(), Equals, "[]")
-	
-	// Clean up
-	resp, err = s.HTTPRequest("DELETE", "/api/repos/dummy?force=1", nil)
-	c.Assert(err, IsNil)
-	c.Check(resp.Code, Equals, 200)
 }

api/publish.go

@@ -16,8 +16,8 @@ import (
 type signingParams struct {
 	// Don't sign published repository
 	Skip bool `                json:"Skip"           example:"false"`
-	// GPG key ID to use when signing the release, if not specified default key is used
-	GpgKey string `            json:"GpgKey"         example:"A0546A43624A8331"`
+	// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used
+	GpgKey string `            json:"GpgKey"         example:"KEY_ID_a, KEY_ID_b"`
 	// GPG keyring to use (instead of default)
 	Keyring string `           json:"Keyring"        example:"trustedkeys.gpg"`
 	// GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -41,7 +41,21 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(options.GpgKey)
+
+	var multiGpgKeys []string
+	// REST params have priority over config
+	if options.GpgKey != "" {
+		for _, p := range strings.Split(options.GpgKey, ",") {
+			if t := strings.TrimSpace(p); t != "" {
+				multiGpgKeys = append(multiGpgKeys, t)
+			}
+		}
+	} else if len(context.Config().GpgKeys) > 0 {
+		multiGpgKeys = context.Config().GpgKeys
+	}
+	for _, gpgKey := range multiGpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(options.Keyring, options.SecretKeyring)
 	signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
 
@@ -110,7 +124,7 @@ func apiPublishList(c *gin.Context) {
 // @Description See also: `aptly publish show`
 // @Tags Publish
 // @Produce json
-// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs"
+// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs"
 // @Param distribution path string true "distribution name"
 // @Success 200 {object} deb.PublishedRepo
 // @Failure 404 {object} Error "Published repository not found"
@@ -146,10 +160,6 @@ type publishedRepoCreateParams struct {
 	Sources []sourceParams `binding:"required"    json:"Sources"`
 	// Distribution name, if missing Aptly would try to guess from sources
 	Distribution string `                         json:"Distribution"          example:"bookworm"`
-	// Value of Label: field in published repository stanza
-	Label string `                                json:"Label"                 example:""`
-	// Value of Origin: field in published repository stanza
-	Origin string `                               json:"Origin"                example:""`
 	// when publishing, overwrite files in pool/ directory without notice
 	ForceOverwrite bool `                         json:"ForceOverwrite"        example:"false"`
 	// Override list of published architectures
@@ -182,7 +192,7 @@ type publishedRepoCreateParams struct {
 // @Description **Example:**
 // @Description ```
 // @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos
-// @Description {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
+// @Description {"Architectures":["i386"],"Distribution":"wheezy","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""}
 // @Description ```
 // @Description
 // @Description See also: `aptly publish create`
@@ -249,7 +259,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 				return
 			}
 
-			resources = append(resources, string(snapshot.ResourceKey()))
+			resources = append(resources, string(snapshot.Key()))
 			sources = append(sources, snapshot)
 		}
 	} else if b.SourceKind == deb.SourceLocalRepo {
@@ -280,11 +290,24 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		multiDist = *b.MultiDist
 	}
 
-	collection := collectionFactory.PublishedRepoCollection()
+	// Non-MultiDist publishes share a single pool/ directory under the
+	// prefix.  Lock at the prefix level so that concurrent publish/drop
+	// operations on sibling distributions cannot race during cleanup.
+	if !multiDist {
+		storagePrefix := prefix
+		if storage != "" {
+			storagePrefix = storage + ":" + prefix
+		}
+
+		resources = append(resources, deb.PrefixPoolLockKey(storagePrefix))
+	}
 
 	taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"",
 		b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`))
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
 		taskDetail := task.PublishDetail{
 			Detail: detail,
 		}
@@ -296,10 +319,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 		for _, source := range sources {
 			switch s := source.(type) {
 			case *deb.Snapshot:
-				snapshotCollection := collectionFactory.SnapshotCollection()
+				snapshotCollection := taskCollectionFactory.SnapshotCollection()
 				err = snapshotCollection.LoadComplete(s)
 			case *deb.LocalRepo:
-				localCollection := collectionFactory.LocalRepoCollection()
+				localCollection := taskCollectionFactory.LocalRepoCollection()
 				err = localCollection.LoadComplete(s)
 			default:
 				err = fmt.Errorf("unexpected type for source: %T", source)
@@ -309,23 +332,17 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			}
 		}
 
-		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist)
+		published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, taskCollectionFactory, multiDist)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		resources = append(resources, string(published.Key()))
-
-		if b.Origin != "" {
-			published.Origin = b.Origin
-		}
 		if b.NotAutomatic != "" {
 			published.NotAutomatic = b.NotAutomatic
 		}
 		if b.ButAutomaticUpgrades != "" {
 			published.ButAutomaticUpgrades = b.ButAutomaticUpgrades
 		}
-		published.Label = b.Label
 
 		published.SkipContents = context.Config().SkipContentsPublishing
 		if b.SkipContents != nil {
@@ -341,18 +358,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
 			published.AcquireByHash = *b.AcquireByHash
 		}
 
-		duplicate := collection.CheckDuplicate(published)
+		duplicate := taskCollection.CheckDuplicate(published)
 		if duplicate != nil {
-			_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
+			_ = taskCollectionFactory.PublishedRepoCollection().LoadComplete(duplicate, taskCollectionFactory)
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
 		}
 
-		err = collection.Add(published)
+		err = taskCollection.Add(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -378,13 +395,6 @@ type publishedRepoUpdateSwitchParams struct {
 	AcquireByHash *bool `                         json:"AcquireByHash"  example:"false"`
 	// Enable multiple packages with the same filename in different distributions
 	MultiDist *bool `                             json:"MultiDist"      example:"false"`
-	// Metadata fields (optional) - if provided, will update the published repository metadata
-	Origin               *string `              json:"Origin,omitempty"`
-	Label                *string `              json:"Label,omitempty"`
-	Suite                *string `              json:"Suite,omitempty"`
-	Codename             *string `              json:"Codename,omitempty"`
-	NotAutomatic         *string `              json:"NotAutomatic,omitempty"`
-	ButAutomaticUpgrades *string `              json:"ButAutomaticUpgrades,omitempty"`
 }
 
 // @Summary Update Published Repository
@@ -400,7 +410,6 @@ type publishedRepoUpdateSwitchParams struct {
 // @Description
 // @Description See also: `aptly publish update` / `aptly publish switch`
 // @Tags Publish
-// @Produce json
 // @Param prefix path string true "publishing prefix"
 // @Param distribution path string true "distribution name"
 // @Param _async query bool false "Run in background and return task object"
@@ -432,6 +441,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
 
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
@@ -439,68 +449,76 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		return
 	}
 
+	resources := []string{string(published.Key())}
+
 	if published.SourceKind == deb.SourceLocalRepo {
 		if len(b.Snapshots) > 0 {
 			AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
 			return
 		}
-	} else if published.SourceKind == deb.SourceSnapshot {
-		for _, snapshotInfo := range b.Snapshots {
-			_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
 			if err2 != nil {
 				AbortWithJSONError(c, http.StatusNotFound, err2)
 				return
 			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, snapshotInfo := range b.Snapshots {
+			snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
 		}
 	} else {
 		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
 		return
 	}
 
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
 	}
 
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
-	// Update metadata fields if provided
-	if b.Origin != nil {
-		published.Origin = *b.Origin
-	}
-	if b.Label != nil {
-		published.Label = *b.Label
-	}
-	if b.Suite != nil {
-		published.Suite = *b.Suite
-	}
-	if b.Codename != nil {
-		published.Codename = *b.Codename
-	}
-	if b.NotAutomatic != nil {
-		published.NotAutomatic = *b.NotAutomatic
-	}
-	if b.ButAutomaticUpgrades != nil {
-		published.ButAutomaticUpgrades = *b.ButAutomaticUpgrades
-	}
-
-	resources := []string{string(published.Key())}
+	// Field mutations and fresh DB load are deferred to inside the task so
+	// they always operate on a consistent state after the lock is held.
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(published, collectionFactory)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
+		}
+
 		revision := published.ObtainRevision()
 		sources := revision.Sources
 
@@ -512,17 +530,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 			}
 		}
 
-		result, err := published.Update(collectionFactory, out)
+		result, err := published.Update(taskCollectionFactory, out)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -530,10 +548,19 @@ func apiPublishUpdateSwitch(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil
@@ -578,10 +605,19 @@ func apiPublishDrop(c *gin.Context) {
 	}
 
 	resources := []string{string(published.Key())}
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that a drop cannot race
+	// with a concurrent update or drop of a sibling distribution during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
 	taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.Remove(context, storage, prefix, distribution,
-			collectionFactory, out, force, skipCleanup)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		err := taskCollection.Remove(context, storage, prefix, distribution,
+			taskCollectionFactory, out, force, skipCleanup)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
 		}
@@ -617,43 +653,52 @@ func apiPublishAddSource(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly (no LoadComplete) to verify existence and obtain the
+	// resource key and task name.  The actual mutation is performed inside
+	// the task on a freshly loaded copy to prevent lost-update races.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	component := b.Component
-	name := b.Name
-
-	_, exists := sources[component]
-	if exists {
-		AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component))
-		return
-	}
-
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		component := b.Component
+		name := b.Name
+
+		_, exists := sources[component]
+		if exists {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("unable to create: Component '%s' already exists", component)
+		}
+
+		sources[component] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -735,39 +780,48 @@ func apiPublishSetSources(c *gin.Context) {
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
 
+	if c.Bind(&b) != nil {
+		return
+	}
+
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := make(map[string]string, len(b))
-	revision.Sources = sources
-
-	for _, source := range b {
-		component := source.Component
-		name := source.Name
-		sources[component] = name
-	}
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := make(map[string]string, len(b))
+		revision.Sources = sources
+
+		for _, source := range b {
+			component := source.Component
+			name := source.Name
+			sources[component] = name
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -800,24 +854,33 @@ func apiPublishDropChanges(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and DropRevision happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	published.DropRevision()
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		published.DropRevision()
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -853,51 +916,58 @@ func apiPublishUpdateSource(c *gin.Context) {
 	param := slashEscape(c.Params.ByName("prefix"))
 	storage, prefix := deb.ParsePrefix(param)
 	distribution := slashEscape(c.Params.ByName("distribution"))
-	component := slashEscape(c.Params.ByName("component"))
+	urlComponent := slashEscape(c.Params.ByName("component"))
+
+	// Default component to the URL path segment; the body may rename it.
+	b.Component = urlComponent
+	if c.Bind(&b) != nil {
+		return
+	}
 
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component))
-		return
-	}
-
-	b.Component = component
-	b.Name = revision.Sources[component]
-
-	if c.Bind(&b) != nil {
-		return
-	}
-
-	if b.Component != component {
-		delete(sources, component)
-	}
-
-	component = b.Component
-	name := b.Name
-	sources[component] = name
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[urlComponent]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: Component '%s' does not exist", urlComponent)
+		}
+
+		if b.Component != urlComponent {
+			delete(sources, urlComponent)
+		}
+
+		newComponent := b.Component
+		name := b.Name
+		sources[newComponent] = name
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -934,33 +1004,41 @@ func apiPublishRemoveSource(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and mutation happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
-		return
-	}
-
-	revision := published.ObtainRevision()
-	sources := revision.Sources
-
-	_, exists := sources[component]
-	if !exists {
-		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component))
-		return
-	}
-
-	delete(sources, component)
-
 	resources := []string{string(published.Key())}
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.Update(published)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
+		}
+
+		revision := published.ObtainRevision()
+		sources := revision.Sources
+
+		_, exists := sources[component]
+		if !exists {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: Component '%s' does not exist", component)
+		}
+
+		delete(sources, component)
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -1024,48 +1102,92 @@ func apiPublishUpdate(c *gin.Context) {
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.PublishedRepoCollection()
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full load and field mutations happen inside the task.
 	published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
 	if err != nil {
 		AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
 		return
 	}
 
-	err = collection.LoadComplete(published, collectionFactory)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
-		return
-	}
-
-	if b.SkipContents != nil {
-		published.SkipContents = *b.SkipContents
-	}
-
-	if b.SkipBz2 != nil {
-		published.SkipBz2 = *b.SkipBz2
-	}
-
-	if b.AcquireByHash != nil {
-		published.AcquireByHash = *b.AcquireByHash
-	}
-
-	if b.MultiDist != nil {
-		published.MultiDist = *b.MultiDist
-	}
-
 	resources := []string{string(published.Key())}
+
+	// Non-MultiDist distributions share a single pool/ directory under the
+	// prefix.  Acquire the prefix-level pool lock so that concurrent updates
+	// on sibling distributions are serialised and cannot race during cleanup.
+	if !published.MultiDist {
+		resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix()))
+	}
+
+	// Lock source repos / snapshots the same way apiPublishUpdateSwitch does,
+	// because published.Update() reads from them and concurrent modification
+	// would produce an inconsistent view.
+	snapshotCollection := collectionFactory.SnapshotCollection()
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+
+	if published.SourceKind == deb.SourceLocalRepo {
+		for _, uuid := range published.Sources {
+			repo, err2 := localRepoCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(repo.Key()))
+		}
+	} else if published.SourceKind == deb.SourceSnapshot {
+		for _, uuid := range published.Sources {
+			snapshot, err2 := snapshotCollection.ByUUID(uuid)
+			if err2 != nil {
+				AbortWithJSONError(c, http.StatusNotFound, err2)
+				return
+			}
+			resources = append(resources, string(snapshot.Key()))
+		}
+	}
+
 	taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		result, err := published.Update(collectionFactory, out)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		err = taskCollection.LoadComplete(published, taskCollectionFactory)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 		}
 
-		err = collection.Update(published)
+		// Capture MultiDist before mutations to detect a false→true transition.
+		prevMultiDist := published.MultiDist
+
+		// Apply field mutations on the freshly loaded object.
+		if b.SkipContents != nil {
+			published.SkipContents = *b.SkipContents
+		}
+		if b.SkipBz2 != nil {
+			published.SkipBz2 = *b.SkipBz2
+		}
+		if b.AcquireByHash != nil {
+			published.AcquireByHash = *b.AcquireByHash
+		}
+		if b.MultiDist != nil {
+			published.MultiDist = *b.MultiDist
+		}
+
+		result, err := published.Update(taskCollectionFactory, out)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+		}
+
+		err = taskCollection.Update(published)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
 		}
@@ -1073,10 +1195,19 @@ func apiPublishUpdate(c *gin.Context) {
 		if b.SkipCleanup == nil || !*b.SkipCleanup {
 			cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
 			cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
-			err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
+			err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
 			}
+			// When MultiDist is toggled, the old pool layout still has files that
+			// CleanupPrefixComponentFiles won't touch (it only scans the new layout).
+			// Run a second pass over the previous layout to remove stale files.
+			if prevMultiDist != published.MultiDist {
+				err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err)
+				}
+			}
 		}
 
 		return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil

api/publish_test.go

@@ -1,675 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-
-	"github.com/aptly-dev/aptly/deb"
-	. "gopkg.in/check.v1"
-)
-
-type PublishAPITestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&PublishAPITestSuite{})
-
-func (s *PublishAPITestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *PublishAPITestSuite) TestPublishList(c *C) {
-	// Test listing published repositories
-	req, _ := http.NewRequest("GET", "/api/publish", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	var result []*deb.PublishedRepo
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(result, NotNil)
-}
-
-func (s *PublishAPITestSuite) TestPublishShow(c *C) {
-	// Test showing a specific published repository
-	// First, we need to create a snapshot and publish it
-	// For now, test the endpoint structure
-	req, _ := http.NewRequest("GET", "/api/publish/test/bookworm", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishUpdate(c *C) {
-	// Test updating a published repository
-	params := struct {
-		Signing signingParams `json:"Signing"`
-	}{
-		Signing: signingParams{Skip: true},
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("PUT", "/api/publish/test/bookworm", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishDrop(c *C) {
-	// Test dropping a published repository
-	req, _ := http.NewRequest("DELETE", "/api/publish/test/bookworm", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishListChanges(c *C) {
-	// Test listing changes in a published repository
-	req, _ := http.NewRequest("GET", "/api/publish/test/bookworm/sources", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishAddSource(c *C) {
-	// Test adding a source to published repository
-	params := sourceParams{
-		Component: "contrib",
-		Name:      "test-snap2",
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/publish/test/bookworm/sources", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishUpdateSource(c *C) {
-	// Test updating a source in published repository
-	params := sourceParams{
-		Component: "main",
-		Name:      "updated-snap",
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("PUT", "/api/publish/test/bookworm/sources/main", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishRemoveSource(c *C) {
-	// Test removing a source from published repository
-	req, _ := http.NewRequest("DELETE", "/api/publish/test/bookworm/sources/contrib", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishSetSources(c *C) {
-	// Test setting sources for published repository
-	params := struct {
-		Sources []sourceParams `json:"Sources"`
-	}{
-		Sources: []sourceParams{
-			{Component: "main", Name: "snap1"},
-			{Component: "contrib", Name: "snap2"},
-		},
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("PUT", "/api/publish/test/bookworm/sources", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestPublishDropChanges(c *C) {
-	// Test dropping changes from published repository
-	req, _ := http.NewRequest("DELETE", "/api/publish/test/bookworm/sources", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the publish doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *PublishAPITestSuite) TestGetSigner(c *C) {
-	// Test getSigner function
-	// Test with Skip = true
-	skipParams := &signingParams{Skip: true}
-	signer, err := getSigner(skipParams)
-	c.Check(err, IsNil)
-	c.Check(signer, IsNil) // Should return nil when Skip is true
-
-	// Test with Skip = false - will use context signer
-	params := &signingParams{
-		Skip:           false,
-		GpgKey:         "A0546A43624A8331",
-		Keyring:        "trustedkeys.gpg",
-		SecretKeyring:  "secretkeys.gpg",
-		Passphrase:     "test",
-		PassphraseFile: "/tmp/passphrase",
-	}
-	signer, err = getSigner(params)
-	c.Check(err, IsNil)
-	c.Check(signer, NotNil)
-}
-
-
-func (s *PublishAPITestSuite) TestSigningParamsStruct(c *C) {
-	// Test signingParams struct and JSON marshaling/unmarshaling
-	params := signingParams{
-		Skip:           true,
-		GpgKey:         "A0546A43624A8331",
-		Keyring:        "trustedkeys.gpg",
-		SecretKeyring:  "secretkeys.gpg",
-		Passphrase:     "verysecure",
-		PassphraseFile: "/etc/aptly.passphrase",
-	}
-
-	// Test JSON marshaling
-	jsonData, err := json.Marshal(params)
-	c.Check(err, IsNil)
-	c.Check(string(jsonData), Matches, ".*Skip.*true.*")
-	c.Check(string(jsonData), Matches, ".*GpgKey.*A0546A43624A8331.*")
-
-	// Test JSON unmarshaling
-	var unmarshaled signingParams
-	err = json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(err, IsNil)
-	c.Check(unmarshaled.Skip, Equals, true)
-	c.Check(unmarshaled.GpgKey, Equals, "A0546A43624A8331")
-	c.Check(unmarshaled.Keyring, Equals, "trustedkeys.gpg")
-	c.Check(unmarshaled.SecretKeyring, Equals, "secretkeys.gpg")
-	c.Check(unmarshaled.Passphrase, Equals, "verysecure")
-	c.Check(unmarshaled.PassphraseFile, Equals, "/etc/aptly.passphrase")
-}
-
-func (s *PublishAPITestSuite) TestSourceParamsStruct(c *C) {
-	// Test sourceParams struct and JSON marshaling/unmarshaling
-	params := sourceParams{
-		Component: "main",
-		Name:      "snap1",
-	}
-
-	// Test JSON marshaling
-	jsonData, err := json.Marshal(params)
-	c.Check(err, IsNil)
-	c.Check(string(jsonData), Matches, ".*Component.*main.*")
-	c.Check(string(jsonData), Matches, ".*Name.*snap1.*")
-
-	// Test JSON unmarshaling
-	var unmarshaled sourceParams
-	err = json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(err, IsNil)
-	c.Check(unmarshaled.Component, Equals, "main")
-	c.Check(unmarshaled.Name, Equals, "snap1")
-}
-
-func (s *PublishAPITestSuite) TestGetSignerSkip(c *C) {
-	// Test getSigner with Skip=true
-	options := &signingParams{
-		Skip: true,
-	}
-
-	signer, err := getSigner(options)
-	c.Check(err, IsNil)
-	c.Check(signer, IsNil)
-}
-
-func (s *PublishAPITestSuite) TestSlashEscape(c *C) {
-	// Test slashEscape function
-	testCases := []struct {
-		input    string
-		expected string
-	}{
-		{"", "."},
-		{"test_path", "test/path"},
-		{"test__path", "test_path"},
-		{"test_path_file", "test/path/file"},
-		{"test__test__test", "test_test_test"},
-		{"_test_", "test/"},
-		{"__", "_"},
-		{"test_path__with__underscores", "test/path_with_underscores"},
-		{"complex_path__example_test", "complex/path_example/test"},
-	}
-
-	for _, tc := range testCases {
-		result := slashEscape(tc.input)
-		c.Check(result, Equals, tc.expected, Commentf("Input: %s", tc.input))
-	}
-}
-
-func (s *PublishAPITestSuite) TestSlashEscapeEdgeCases(c *C) {
-	// Test edge cases for slashEscape
-	edgeCases := []struct {
-		input    string
-		expected string
-	}{
-		{"simple", "simple"},
-		{"no_underscores_here", "no/underscores/here"},
-		{"double__only", "double_only"},
-		{"_", "."},
-		{"__only", "_only"},
-		{"only_", "only/"},
-		{"mixed_case__Test_Path", "mixed/case_Test/Path"},
-		{"numbers_123__test", "numbers/123_test"},
-		{"special-chars.test_path", "special-chars.test/path"},
-	}
-
-	for _, tc := range edgeCases {
-		result := slashEscape(tc.input)
-		c.Check(result, Equals, tc.expected, Commentf("Input: '%s'", tc.input))
-	}
-}
-
-func (s *PublishAPITestSuite) TestApiPublishListBasic(c *C) {
-	// Test basic API publish list endpoint
-	req, _ := http.NewRequest("GET", "/api/publish", nil)
-	w := httptest.NewRecorder()
-
-	// Now context is set up properly through APISuite
-	s.router.ServeHTTP(w, req)
-	// Should return OK with empty list
-	c.Check(w.Code, Equals, http.StatusOK)
-}
-
-func (s *PublishAPITestSuite) TestApiPublishShowBasic(c *C) {
-	// Test basic API publish show endpoint
-	req, _ := http.NewRequest("GET", "/api/publish/test-prefix/test-dist", nil)
-	w := httptest.NewRecorder()
-
-	// This will fail because context is not set up properly
-	s.router.ServeHTTP(w, req)
-	// Expect some kind of error due to missing context
-	c.Check(w.Code, Not(Equals), http.StatusOK)
-}
-
-func (s *PublishAPITestSuite) TestApiPublishShowWithSlashEscape(c *C) {
-	// Test API publish show with slash escape characters
-	req, _ := http.NewRequest("GET", "/api/publish/test__prefix/test_dist", nil)
-	w := httptest.NewRecorder()
-
-	s.router.ServeHTTP(w, req)
-	// Should attempt to process the escaped path
-	c.Check(w.Code, Not(Equals), http.StatusOK) // Expected to fail due to missing context
-}
-
-func (s *PublishAPITestSuite) TestPublishedRepoCreateParamsStruct(c *C) {
-	// Test publishedRepoCreateParams struct
-	skipContents := true
-	skipCleanup := false
-	skipBz2 := true
-	acquireByHash := false
-	multiDist := true
-
-	params := publishedRepoCreateParams{
-		SourceKind:     "snapshot",
-		Sources:        []sourceParams{{Component: "main", Name: "test-snap"}},
-		Distribution:   "bookworm",
-		Label:          "Test Label",
-		Origin:         "Test Origin",
-		ForceOverwrite: true,
-		Architectures:  []string{"amd64", "armhf"},
-		Signing: signingParams{
-			Skip:   false,
-			GpgKey: "A0546A43624A8331",
-		},
-		NotAutomatic:         "yes",
-		ButAutomaticUpgrades: "yes",
-		SkipContents:         &skipContents,
-		SkipCleanup:          &skipCleanup,
-		SkipBz2:              &skipBz2,
-		AcquireByHash:        &acquireByHash,
-		MultiDist:            &multiDist,
-	}
-
-	// Test JSON marshaling
-	jsonData, err := json.Marshal(params)
-	c.Check(err, IsNil)
-	c.Check(string(jsonData), Matches, ".*SourceKind.*snapshot.*")
-	c.Check(string(jsonData), Matches, ".*Distribution.*bookworm.*")
-	c.Check(string(jsonData), Matches, ".*Label.*Test Label.*")
-	c.Check(string(jsonData), Matches, ".*Origin.*Test Origin.*")
-	c.Check(string(jsonData), Matches, ".*ForceOverwrite.*true.*")
-
-	// Test JSON unmarshaling
-	var unmarshaled publishedRepoCreateParams
-	err = json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(err, IsNil)
-	c.Check(unmarshaled.SourceKind, Equals, "snapshot")
-	c.Check(unmarshaled.Distribution, Equals, "bookworm")
-	c.Check(unmarshaled.Label, Equals, "Test Label")
-	c.Check(unmarshaled.Origin, Equals, "Test Origin")
-	c.Check(unmarshaled.ForceOverwrite, Equals, true)
-	c.Check(len(unmarshaled.Sources), Equals, 1)
-	c.Check(unmarshaled.Sources[0].Component, Equals, "main")
-	c.Check(unmarshaled.Sources[0].Name, Equals, "test-snap")
-	c.Check(len(unmarshaled.Architectures), Equals, 2)
-	c.Check(unmarshaled.Architectures[0], Equals, "amd64")
-	c.Check(unmarshaled.Architectures[1], Equals, "armhf")
-	c.Check(*unmarshaled.SkipContents, Equals, true)
-	c.Check(*unmarshaled.SkipCleanup, Equals, false)
-	c.Check(*unmarshaled.SkipBz2, Equals, true)
-	c.Check(*unmarshaled.AcquireByHash, Equals, false)
-	c.Check(*unmarshaled.MultiDist, Equals, true)
-}
-
-func (s *PublishAPITestSuite) TestPublishedRepoUpdateSwitchParamsStruct(c *C) {
-	// Test publishedRepoUpdateSwitchParams struct
-	skipContents := false
-	skipBz2 := true
-	skipCleanup := true
-	acquireByHash := true
-	multiDist := false
-
-	params := publishedRepoUpdateSwitchParams{
-		ForceOverwrite: true,
-		Signing: signingParams{
-			Skip:    true,
-			GpgKey:  "testkey",
-			Keyring: "test.gpg",
-		},
-		SkipContents:  &skipContents,
-		SkipBz2:       &skipBz2,
-		SkipCleanup:   &skipCleanup,
-		Snapshots:     []sourceParams{{Component: "main", Name: "snap1"}, {Component: "contrib", Name: "snap2"}},
-		AcquireByHash: &acquireByHash,
-		MultiDist:     &multiDist,
-	}
-
-	// Test JSON marshaling
-	jsonData, err := json.Marshal(params)
-	c.Check(err, IsNil)
-	c.Check(string(jsonData), Matches, ".*ForceOverwrite.*true.*")
-	c.Check(string(jsonData), Matches, ".*SkipContents.*false.*")
-	c.Check(string(jsonData), Matches, ".*SkipBz2.*true.*")
-
-	// Test JSON unmarshaling
-	var unmarshaled publishedRepoUpdateSwitchParams
-	err = json.Unmarshal(jsonData, &unmarshaled)
-	c.Check(err, IsNil)
-	c.Check(unmarshaled.ForceOverwrite, Equals, true)
-	c.Check(unmarshaled.Signing.Skip, Equals, true)
-	c.Check(unmarshaled.Signing.GpgKey, Equals, "testkey")
-	c.Check(unmarshaled.Signing.Keyring, Equals, "test.gpg")
-	c.Check(*unmarshaled.SkipContents, Equals, false)
-	c.Check(*unmarshaled.SkipBz2, Equals, true)
-	c.Check(*unmarshaled.SkipCleanup, Equals, true)
-	c.Check(*unmarshaled.AcquireByHash, Equals, true)
-	c.Check(*unmarshaled.MultiDist, Equals, false)
-	c.Check(len(unmarshaled.Snapshots), Equals, 2)
-	c.Check(unmarshaled.Snapshots[0].Component, Equals, "main")
-	c.Check(unmarshaled.Snapshots[0].Name, Equals, "snap1")
-	c.Check(unmarshaled.Snapshots[1].Component, Equals, "contrib")
-	c.Check(unmarshaled.Snapshots[1].Name, Equals, "snap2")
-}
-
-func (s *PublishAPITestSuite) TestApiPublishRepoOrSnapshotInvalidJSON(c *C) {
-	// Test POST endpoint with invalid JSON
-	req, _ := http.NewRequest("POST", "/api/publish/test-prefix", strings.NewReader("invalid json"))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, http.StatusBadRequest)
-}
-
-func (s *PublishAPITestSuite) TestApiPublishRepoOrSnapshotEmptySources(c *C) {
-	// Test POST endpoint with empty sources
-	params := publishedRepoCreateParams{
-		SourceKind:   "snapshot",
-		Sources:      []sourceParams{}, // Empty sources
-		Distribution: "test",
-	}
-
-	jsonData, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/publish/test-prefix", bytes.NewReader(jsonData))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should return 400 due to empty sources
-	c.Check(w.Code, Equals, http.StatusBadRequest)
-}
-
-func (s *PublishAPITestSuite) TestApiPublishRepoOrSnapshotUnknownSourceKind(c *C) {
-	// Test POST endpoint with unknown source kind
-	params := publishedRepoCreateParams{
-		SourceKind:   "unknown",
-		Sources:      []sourceParams{{Component: "main", Name: "test"}},
-		Distribution: "test",
-	}
-
-	jsonData, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/publish/test-prefix", bytes.NewReader(jsonData))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should return 400 due to unknown source kind
-	c.Check(w.Code, Equals, http.StatusBadRequest)
-}
-
-func (s *PublishAPITestSuite) TestApiPublishRepoOrSnapshotValidRequest(c *C) {
-	// Test POST endpoint with valid request (will fail due to missing context)
-	params := publishedRepoCreateParams{
-		SourceKind:   deb.SourceSnapshot,
-		Sources:      []sourceParams{{Component: "main", Name: "test-snap"}},
-		Distribution: "test-dist",
-		Signing:      signingParams{Skip: true},
-	}
-
-	jsonData, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/publish/test-prefix", bytes.NewReader(jsonData))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will fail due to missing context but should get past basic validation
-	c.Check(w.Code, Not(Equals), http.StatusBadRequest) // Should not be a 400 error
-}
-
-func (s *PublishAPITestSuite) TestApiPublishRepoOrSnapshotLocalRepoSourceKind(c *C) {
-	// Test POST endpoint with local repo source kind
-	params := publishedRepoCreateParams{
-		SourceKind:   deb.SourceLocalRepo,
-		Sources:      []sourceParams{{Component: "main", Name: "test-repo"}},
-		Distribution: "test-dist",
-		Signing:      signingParams{Skip: true},
-	}
-
-	jsonData, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/publish/test-prefix", bytes.NewReader(jsonData))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will fail due to missing context but should get past basic validation
-	c.Check(w.Code, Not(Equals), http.StatusBadRequest) // Should not be a 400 error
-}
-
-func (s *PublishAPITestSuite) TestSigningParamsEdgeCases(c *C) {
-	// Test signingParams with edge cases
-	testCases := []signingParams{
-		{Skip: true}, // Minimal case
-		{Skip: false, GpgKey: "", Keyring: "", SecretKeyring: "", Passphrase: "", PassphraseFile: ""},                   // Empty strings
-		{Skip: false, GpgKey: "very-long-key-id-123456789012345678901234567890", Keyring: "very-long-keyring-name.gpg"}, // Long values
-		{Skip: false, Passphrase: "password with spaces and special chars !@#$%^&*()"},                                  // Special characters
-		{Skip: false, PassphraseFile: "/very/long/path/to/passphrase/file/that/might/not/exist.txt"},                    // Long file path
-	}
-
-	for i, params := range testCases {
-		// Test JSON marshaling/unmarshaling
-		jsonData, err := json.Marshal(params)
-		c.Check(err, IsNil, Commentf("Test case %d", i))
-
-		var unmarshaled signingParams
-		err = json.Unmarshal(jsonData, &unmarshaled)
-		c.Check(err, IsNil, Commentf("Test case %d", i))
-		c.Check(unmarshaled.Skip, Equals, params.Skip, Commentf("Test case %d", i))
-		c.Check(unmarshaled.GpgKey, Equals, params.GpgKey, Commentf("Test case %d", i))
-		c.Check(unmarshaled.Keyring, Equals, params.Keyring, Commentf("Test case %d", i))
-	}
-}
-
-func (s *PublishAPITestSuite) TestSourceParamsEdgeCases(c *C) {
-	// Test sourceParams with edge cases
-	testCases := []sourceParams{
-		{Component: "", Name: ""}, // Empty strings
-		{Component: "very-long-component-name-with-dashes-and-numbers-123", Name: "very-long-name-456"}, // Long values
-		{Component: "comp.with.dots", Name: "name_with_underscores"},                                    // Special characters
-		{Component: "UPPERCASE", Name: "MixedCase"},                                                     // Case variations
-		{Component: "123numeric", Name: "456numbers"},                                                   // Numeric values
-	}
-
-	for i, params := range testCases {
-		// Test JSON marshaling/unmarshaling
-		jsonData, err := json.Marshal(params)
-		c.Check(err, IsNil, Commentf("Test case %d", i))
-
-		var unmarshaled sourceParams
-		err = json.Unmarshal(jsonData, &unmarshaled)
-		c.Check(err, IsNil, Commentf("Test case %d", i))
-		c.Check(unmarshaled.Component, Equals, params.Component, Commentf("Test case %d", i))
-		c.Check(unmarshaled.Name, Equals, params.Name, Commentf("Test case %d", i))
-	}
-}
-
-func (s *PublishAPITestSuite) TestSlashEscapeComprehensive(c *C) {
-	// Comprehensive test of slashEscape function
-	testCases := []struct {
-		input       string
-		expected    string
-		description string
-	}{
-		{"", ".", "empty string"},
-		{"simple", "simple", "no underscores"},
-		{"one_underscore", "one/underscore", "single underscore"},
-		{"two__underscores", "two_underscores", "double underscore"},
-		{"_leading", "leading", "leading underscore"},
-		{"trailing_", "trailing/", "trailing underscore"},
-		{"_both_", "both/", "both leading and trailing"},
-		{"__double_leading", "_double/leading", "double leading underscore"},
-		{"trailing_double__", "trailing/double_", "double trailing underscore"},
-		{"mixed_single__double_combo", "mixed/single_double/combo", "mixed single and double"},
-		{"complex_path__with_multiple__sections", "complex/path_with/multiple_sections", "complex path"},
-		{"a_b_c_d_e", "a/b/c/d/e", "multiple single underscores"},
-		{"a__b__c__d__e", "a_b_c_d_e", "multiple double underscores"},
-		{"_a__b_c__d_", "a_b/c_d/", "mixed pattern"},
-		{"test___triple", "test_/triple", "triple underscore"},
-		{"test____quad", "test__quad", "quadruple underscore"},
-	}
-
-	for _, tc := range testCases {
-		result := slashEscape(tc.input)
-		c.Check(result, Equals, tc.expected, Commentf("Test case: %s (input: '%s')", tc.description, tc.input))
-	}
-}
-
-// Mock implementations for testing context dependencies
-type MockSigner struct {
-	initError      error
-	key            string
-	keyring        string
-	secretKeyring  string
-	passphrase     string
-	passphraseFile string
-	batch          bool
-}
-
-func (m *MockSigner) SetKey(key string) { m.key = key }
-func (m *MockSigner) SetKeyRing(keyring, secretKeyring string) {
-	m.keyring = keyring
-	m.secretKeyring = secretKeyring
-}
-func (m *MockSigner) SetPassphrase(passphrase, passphraseFile string) {
-	m.passphrase = passphrase
-	m.passphraseFile = passphraseFile
-}
-func (m *MockSigner) SetBatch(batch bool) { m.batch = batch }
-func (m *MockSigner) Init() error         { return m.initError }
-
-func (s *PublishAPITestSuite) TestGetSignerMockSuccess(c *C) {
-	// Test getSigner logic with mock (can't test actual getSigner due to context dependencies)
-	options := &signingParams{
-		Skip:           false,
-		GpgKey:         "testkey",
-		Keyring:        "test.gpg",
-		SecretKeyring:  "secret.gpg",
-		Passphrase:     "testpass",
-		PassphraseFile: "/tmp/passfile",
-	}
-
-	// Mock the signer behavior
-	mockSigner := &MockSigner{initError: nil}
-
-	// Simulate what getSigner would do
-	mockSigner.SetKey(options.GpgKey)
-	mockSigner.SetKeyRing(options.Keyring, options.SecretKeyring)
-	mockSigner.SetPassphrase(options.Passphrase, options.PassphraseFile)
-	mockSigner.SetBatch(true)
-	err := mockSigner.Init()
-
-	c.Check(err, IsNil)
-	c.Check(mockSigner.key, Equals, "testkey")
-	c.Check(mockSigner.keyring, Equals, "test.gpg")
-	c.Check(mockSigner.secretKeyring, Equals, "secret.gpg")
-	c.Check(mockSigner.passphrase, Equals, "testpass")
-	c.Check(mockSigner.passphraseFile, Equals, "/tmp/passfile")
-	c.Check(mockSigner.batch, Equals, true)
-}
-
-func (s *PublishAPITestSuite) TestGetSignerMockError(c *C) {
-	// Test getSigner logic with mock error
-	options := &signingParams{
-		Skip:   false,
-		GpgKey: "invalidkey",
-	}
-
-	// Mock the signer behavior with error
-	mockSigner := &MockSigner{initError: fmt.Errorf("mock init error")}
-
-	mockSigner.SetKey(options.GpgKey)
-	mockSigner.SetKeyRing(options.Keyring, options.SecretKeyring)
-	mockSigner.SetPassphrase(options.Passphrase, options.PassphraseFile)
-	mockSigner.SetBatch(true)
-	err := mockSigner.Init()
-
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "mock init error")
-}

api/published_file_missing_test.go

@@ -0,0 +1,737 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/aptly-dev/aptly/aptly"
+	ctx "github.com/aptly-dev/aptly/context"
+	"github.com/aptly-dev/aptly/deb"
+	"github.com/gin-gonic/gin"
+	"github.com/smira/flag"
+
+	. "gopkg.in/check.v1"
+)
+
+// PublishedFileMissingSuite reproduces the exact bug where:
+// - Package import succeeds
+// - Metadata is updated (Packages.gz shows the package)
+// - Publish reports success
+// - BUT the .deb file is missing from the published pool directory
+// - Result: apt-get returns 404 when trying to download the package
+type PublishedFileMissingSuite struct {
+	context    *ctx.AptlyContext
+	flags      *flag.FlagSet
+	configFile *os.File
+	router     http.Handler
+	tempDir    string
+	poolPath   string
+	publicPath string
+}
+
+var _ = Suite(&PublishedFileMissingSuite{})
+
+func (s *PublishedFileMissingSuite) SetUpSuite(c *C) {
+	aptly.Version = "publishedFileMissingTest"
+
+	tempDir, err := os.MkdirTemp("", "aptly-published-missing-test")
+	c.Assert(err, IsNil)
+	s.tempDir = tempDir
+	s.poolPath = filepath.Join(tempDir, "pool")
+	s.publicPath = filepath.Join(tempDir, "public")
+
+	file, err := os.CreateTemp("", "aptly-published-missing-config")
+	c.Assert(err, IsNil)
+	s.configFile = file
+
+	config := gin.H{
+		"rootDir":                    tempDir,
+		"downloadDir":                filepath.Join(tempDir, "download"),
+		"architectures":              []string{"amd64"},
+		"dependencyFollowSuggests":   false,
+		"dependencyFollowRecommends": false,
+		"gpgDisableSign":             true,
+		"gpgDisableVerify":           true,
+		"gpgProvider":                "internal",
+		"skipLegacyPool":             true,
+		"enableMetricsEndpoint":      false,
+	}
+
+	jsonString, err := json.Marshal(config)
+	c.Assert(err, IsNil)
+	_, err = file.Write(jsonString)
+	c.Assert(err, IsNil)
+
+	flags := flag.NewFlagSet("publishedFileMissingTestFlags", flag.ContinueOnError)
+	flags.Bool("no-lock", true, "disable database locking for test")
+	flags.Int("db-open-attempts", 3, "dummy")
+	flags.String("config", s.configFile.Name(), "config file")
+	flags.String("architectures", "", "dummy")
+	s.flags = flags
+
+	context, err := ctx.NewContext(s.flags)
+	c.Assert(err, IsNil)
+
+	s.context = context
+	s.router = Router(context)
+}
+
+func (s *PublishedFileMissingSuite) TearDownSuite(c *C) {
+	if s.configFile != nil {
+		_ = os.Remove(s.configFile.Name())
+	}
+	if s.context != nil {
+		s.context.Shutdown()
+	}
+	if s.tempDir != "" {
+		_ = os.RemoveAll(s.tempDir)
+	}
+}
+
+func (s *PublishedFileMissingSuite) SetUpTest(c *C) {
+	collectionFactory := s.context.NewCollectionFactory()
+
+	localRepoCollection := collectionFactory.LocalRepoCollection()
+	_ = localRepoCollection.ForEach(func(repo *deb.LocalRepo) error {
+		_ = localRepoCollection.Drop(repo)
+		return nil
+	})
+
+	publishedCollection := collectionFactory.PublishedRepoCollection()
+	_ = publishedCollection.ForEach(func(published *deb.PublishedRepo) error {
+		_ = publishedCollection.Remove(s.context, published.Storage, published.Prefix,
+			published.Distribution, collectionFactory, nil, true, true)
+		return nil
+	})
+}
+
+func (s *PublishedFileMissingSuite) TearDownTest(c *C) {
+	s.SetUpTest(c)
+}
+
+func (s *PublishedFileMissingSuite) httpRequest(c *C, method string, url string, body []byte) *httptest.ResponseRecorder {
+	w := httptest.NewRecorder()
+	var req *http.Request
+	var err error
+
+	if body != nil {
+		req, err = http.NewRequest(method, url, bytes.NewReader(body))
+	} else {
+		req, err = http.NewRequest(method, url, nil)
+	}
+	c.Assert(err, IsNil)
+	req.Header.Add("Content-Type", "application/json")
+	s.router.ServeHTTP(w, req)
+	return w
+}
+
+func (s *PublishedFileMissingSuite) createDebPackage(c *C, uploadID, packageName, version string) {
+	uploadPath := s.context.UploadPath()
+	uploadDir := filepath.Join(uploadPath, uploadID)
+	err := os.MkdirAll(uploadDir, 0755)
+	c.Assert(err, IsNil)
+
+	tempDir, err := os.MkdirTemp("", "deb-build")
+	c.Assert(err, IsNil)
+	defer func() { _ = os.RemoveAll(tempDir) }()
+
+	debianDir := filepath.Join(tempDir, "DEBIAN")
+	err = os.MkdirAll(debianDir, 0755)
+	c.Assert(err, IsNil)
+
+	controlContent := fmt.Sprintf(`Package: %s
+Version: %s
+Section: libs
+Priority: optional
+Architecture: amd64
+Maintainer: Test <test@example.com>
+Description: Test package
+ Test package for published file missing bug.
+`, packageName, version)
+
+	err = os.WriteFile(filepath.Join(debianDir, "control"), []byte(controlContent), 0644)
+	c.Assert(err, IsNil)
+
+	usrDir := filepath.Join(tempDir, "usr", "lib")
+	err = os.MkdirAll(usrDir, 0755)
+	c.Assert(err, IsNil)
+	err = os.WriteFile(filepath.Join(usrDir, "lib.so"), []byte("library"), 0644)
+	c.Assert(err, IsNil)
+
+	debFile := filepath.Join(uploadDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+	cmd := exec.Command("dpkg-deb", "--build", tempDir, debFile)
+	err = cmd.Run()
+	c.Assert(err, IsNil)
+}
+
+// TestPublishedFileGoMissing reproduces the exact production bug
+func (s *PublishedFileMissingSuite) TestPublishedFileGoMissing(c *C) {
+	c.Log("=== Reproducing: Package in metadata but 404 on download ===")
+
+	// Create and publish a repository
+	repoName := "test-repo"
+	distribution := "bullseye"
+
+	createBody, _ := json.Marshal(gin.H{
+		"Name":                repoName,
+		"DefaultDistribution": distribution,
+		"DefaultComponent":    "main",
+	})
+	resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to create repo: %s", resp.Body.String()))
+
+	publishBody, _ := json.Marshal(gin.H{
+		"SourceKind":    "local",
+		"Distribution":  distribution,
+		"Architectures": []string{"amd64"},
+		"Sources": []gin.H{
+			{"Component": "main", "Name": repoName},
+		},
+		"Signing": gin.H{"Skip": true},
+	})
+	resp = s.httpRequest(c, "POST", "/api/publish/hrt", publishBody)
+	c.Assert(resp.Code, Equals, 201, Commentf("Failed to publish: %s", resp.Body.String()))
+
+	// Create package
+	packageName := "hrt-libblobbyclient1"
+	version := "20250926.152427+hrtdeb11"
+	uploadID := "test-upload-1"
+
+	s.createDebPackage(c, uploadID, packageName, version)
+
+	// Add package
+	resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package: %s", resp.Body.String()))
+
+	// Update publish
+	updateBody, _ := json.Marshal(gin.H{
+		"Signing":        gin.H{"Skip": true},
+		"ForceOverwrite": true,
+	})
+	resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/hrt/%s", distribution), updateBody)
+	c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String()))
+
+	// Now check if the file is actually accessible in the published location
+	publishedStorage, err := s.context.GetPublishedStorage("")
+	c.Assert(err, IsNil)
+	publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+	// Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb
+	expectedPath := filepath.Join(publicPath, "hrt", "pool", "main", "h", packageName,
+		fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+	c.Logf("Checking for published file at: %s", expectedPath)
+
+	fileInfo, err := os.Stat(expectedPath)
+	fileExists := err == nil
+
+	c.Logf("File exists: %v", fileExists)
+	if fileExists {
+		c.Logf("File size: %d bytes", fileInfo.Size())
+	}
+
+	// Check metadata
+	resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+	var packages []string
+	err = json.Unmarshal(resp.Body.Bytes(), &packages)
+	c.Assert(err, IsNil)
+	c.Logf("Packages in metadata: %d", len(packages))
+
+	// THE BUG: Metadata says package exists, but file is missing from published location
+	if len(packages) > 0 && !fileExists {
+		c.Logf("★★★ BUG REPRODUCED! ★★★")
+		c.Logf("Metadata shows %d package(s) but file is missing at: %s", len(packages), expectedPath)
+		c.Logf("This is exactly what causes: 404 Not Found [IP: 10.20.72.62 3142]")
+
+		c.Fatal("BUG CONFIRMED: Package in metadata but missing from published directory!")
+	}
+
+	c.Assert(fileExists, Equals, true, Commentf(
+		"Published file should exist at %s when package is in metadata", expectedPath))
+}
+
+// TestConcurrentPublishRace tries to trigger the race with concurrent publishes
+func (s *PublishedFileMissingSuite) TestConcurrentPublishRace(c *C) {
+	c.Log("=== Testing concurrent publish race condition ===")
+
+	const numIterations = 4
+
+	for iteration := 0; iteration < numIterations; iteration++ {
+		c.Logf("--- Iteration %d/%d ---", iteration+1, numIterations)
+
+		// Create repo
+		repoName := fmt.Sprintf("race-repo-%d", iteration)
+		distribution := fmt.Sprintf("dist-%d", iteration)
+
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoName,
+			"DefaultDistribution": distribution,
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		publishBody, _ := json.Marshal(gin.H{
+			"SourceKind":    "local",
+			"Distribution":  distribution,
+			"Architectures": []string{"amd64"},
+			"Sources": []gin.H{
+				{"Component": "main", "Name": repoName},
+			},
+			"Signing": gin.H{"Skip": true},
+		})
+		resp = s.httpRequest(c, "POST", "/api/publish/concurrent", publishBody)
+		c.Assert(resp.Code, Equals, 201)
+
+		// Create multiple packages
+		var wg sync.WaitGroup
+		numPackages := 5
+
+		for i := 0; i < numPackages; i++ {
+			wg.Add(1)
+			go func(idx int) {
+				defer wg.Done()
+
+				packageName := fmt.Sprintf("pkg-%d-%d", iteration, idx)
+				version := "1.0.0"
+				uploadID := fmt.Sprintf("upload-%d-%d", iteration, idx)
+
+				s.createDebPackage(c, uploadID, packageName, version)
+
+				// Add package
+				resp := s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil)
+				c.Logf("Package %d add: %d", idx, resp.Code)
+
+				// Small delay
+				time.Sleep(time.Duration(5+idx*2) * time.Millisecond)
+
+				// Publish
+				updateBody, _ := json.Marshal(gin.H{
+					"Signing":        gin.H{"Skip": true},
+					"ForceOverwrite": true,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/concurrent/%s", distribution), updateBody)
+				c.Logf("Publish %d: %d", idx, resp.Code)
+			}(i)
+		}
+
+		wg.Wait()
+		time.Sleep(100 * time.Millisecond)
+
+		// Check all packages
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil)
+		var packages []string
+		err := json.Unmarshal(resp.Body.Bytes(), &packages)
+		c.Assert(err, IsNil)
+
+		// Check published files
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("pkg-%d-%d", iteration, i)
+			version := "1.0.0"
+
+			// Calculate pool path
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, "concurrent", "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, expectedPath)
+			}
+		}
+
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG DETECTED in iteration %d/%d! ★★★", iteration+1, numIterations)
+			c.Logf("Metadata shows %d packages, but %d files are MISSING:", len(packages), len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  [iter %d] File MISSING %d/%d: %s", iteration+1, i+1, len(missingFiles), f)
+			}
+
+			c.Fatalf("BUG REPRODUCED in iteration %d/%d: %d published files missing", iteration+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iteration+1, numIterations, numPackages)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced with current timing", numIterations)
+}
+
+// TestIdenticalPackageRace tests the specific case of identical SHA256 packages
+func (s *PublishedFileMissingSuite) TestIdenticalPackageRace(c *C) {
+	c.Log("=== AGGRESSIVE test: identical package (same SHA256) race ===")
+
+	const numIterations = 4
+	packageName := "shared-package"
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("Iteration %d/%d", iter+1, numIterations)
+
+		// Create two repos that will get the SAME package (unique per iteration)
+		repos := []string{fmt.Sprintf("identical-a-%d", iter), fmt.Sprintf("identical-b-%d", iter)}
+		dists := []string{fmt.Sprintf("dist-a-%d", iter), fmt.Sprintf("dist-b-%d", iter)}
+
+		for i := range repos {
+			createBody, _ := json.Marshal(gin.H{
+				"Name":                repos[i],
+				"DefaultDistribution": dists[i],
+				"DefaultComponent":    "main",
+			})
+			resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+			c.Assert(resp.Code, Equals, 201)
+
+			publishBody, _ := json.Marshal(gin.H{
+				"SourceKind":    "local",
+				"Distribution":  dists[i],
+				"Architectures": []string{"amd64"},
+				"Sources": []gin.H{
+					{"Component": "main", "Name": repos[i]},
+				},
+				"Signing": gin.H{"Skip": true},
+				"SkipBz2": true,
+			})
+			resp = s.httpRequest(c, "POST", "/api/publish/identical", publishBody)
+			c.Assert(resp.Code, Equals, 201)
+		}
+
+		// Create IDENTICAL package file with UNIQUE VERSION per iteration
+		version := fmt.Sprintf("1.0.%d", iter)
+		uploadID1 := fmt.Sprintf("identical-upload-1-%d", iter)
+		uploadID2 := fmt.Sprintf("identical-upload-2-%d", iter)
+
+		s.createDebPackage(c, uploadID1, packageName, version)
+
+		// Copy to second upload (same SHA256)
+		uploadPath := s.context.UploadPath()
+		src := filepath.Join(uploadPath, uploadID1, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+		destDir := filepath.Join(uploadPath, uploadID2)
+		err := os.MkdirAll(destDir, 0755)
+		c.Assert(err, IsNil)
+		dest := filepath.Join(destDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		srcData, readErr := os.ReadFile(src)
+		c.Assert(readErr, IsNil)
+		err = os.WriteFile(dest, srcData, 0644)
+		c.Assert(err, IsNil)
+
+		// Race: add and publish both simultaneously
+		var wg sync.WaitGroup
+		wg.Add(2)
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[0], uploadID1), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[0]), updateBody)
+		}()
+
+		go func() {
+			defer wg.Done()
+			s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[1], uploadID2), nil)
+			updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true})
+			s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[1]), updateBody)
+		}()
+
+		wg.Wait()
+		time.Sleep(200 * time.Millisecond)
+		c.Logf("[iter %d] All operations complete", iter)
+
+		// Check the shared pool location
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		poolSubdir := string(packageName[0])
+		sharedPoolPath := filepath.Join(publicPath, "identical", "pool", "main", poolSubdir, packageName,
+			fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+		fileInfo, err := os.Stat(sharedPoolPath)
+		fileExists := err == nil
+
+		if fileExists {
+			c.Logf("[iter %d] File EXISTS at %s (size: %d)", iter, sharedPoolPath, fileInfo.Size())
+		} else {
+			c.Logf("[iter %d] File MISSING at %s (error: %v)", iter, sharedPoolPath, err)
+		}
+
+		// Check metadata
+		var packagesA, packagesB []string
+		resp := s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[0]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesA)
+		c.Assert(err, IsNil)
+		resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[1]), nil)
+		err = json.Unmarshal(resp.Body.Bytes(), &packagesB)
+		c.Assert(err, IsNil)
+
+		c.Logf("[iter %d] Packages in metadata: A=%d, B=%d", iter, len(packagesA), len(packagesB))
+
+		// THE BUG: Both repos show packages in metadata, but the shared pool file is missing
+		if (len(packagesA) > 0 || len(packagesB) > 0) && !fileExists {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d! ★★★", iter+1)
+			c.Logf("Packages in metadata A: %d, B: %d", len(packagesA), len(packagesB))
+			c.Logf("Shared pool file exists: %v", fileExists)
+			c.Logf("Pool path: %s", sharedPoolPath)
+
+			// List what files ARE in the pool directory
+			poolDir := filepath.Dir(sharedPoolPath)
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("Files in pool directory %s:", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s", entry.Name())
+				}
+			}
+
+			c.Fatalf("Metadata shows packages but shared pool file is missing (iteration %d)", iter+1)
+		}
+	}
+
+	c.Logf("All %d iterations passed - bug not reproduced", numIterations)
+}
+
+// TestConcurrentSnapshotPublishToSamePrefix reproduces the EXACT production bug:
+// Multiple snapshots are published concurrently to the SAME prefix but different distributions.
+// Example from production logs:
+//   - trixie-pgdg published to "external/postgres-auto/trixie"
+//   - bullseye-pgdg published to "external/postgres-auto/bullseye"
+// Both share the same pool directory, causing cleanup race conditions.
+func (s *PublishedFileMissingSuite) TestConcurrentSnapshotPublishToSamePrefix(c *C) {
+	const numIterations = 4
+
+	for iter := 0; iter < numIterations; iter++ {
+		c.Logf("--- Iteration %d/%d ---", iter+1, numIterations)
+
+		// Create two repos with different packages (simulating trixie-pgdg and bullseye-pgdg)
+		repoTrixie := fmt.Sprintf("trixie-pgdg-%d", iter)
+		repoBullseye := fmt.Sprintf("bullseye-pgdg-%d", iter)
+
+		// Create trixie repo
+		createBody, _ := json.Marshal(gin.H{
+			"Name":                repoTrixie,
+			"DefaultDistribution": "trixie",
+			"DefaultComponent":    "main",
+		})
+		resp := s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie repo"))
+
+		// Create bullseye repo
+		createBody, _ = json.Marshal(gin.H{
+			"Name":                repoBullseye,
+			"DefaultDistribution": "bullseye",
+			"DefaultComponent":    "main",
+		})
+		resp = s.httpRequest(c, "POST", "/api/repos", createBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye repo"))
+
+		// Add packages to both repos
+		numPackages := 3
+
+		// Add packages to trixie repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("trixie-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoTrixie, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to trixie"))
+		}
+
+		// Add packages to bullseye repo
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+			uploadID := fmt.Sprintf("bullseye-upload-%d-%d", iter, i)
+
+			s.createDebPackage(c, uploadID, packageName, version)
+			resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoBullseye, uploadID), nil)
+			c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to bullseye"))
+		}
+
+		// Create snapshots from both repos
+		snapshotTrixie := fmt.Sprintf("%s-snap", repoTrixie)
+		snapshotBullseye := fmt.Sprintf("%s-snap", repoBullseye)
+
+		createSnapshotBody, _ := json.Marshal(gin.H{"Name": snapshotTrixie})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoTrixie), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie snapshot"))
+
+		createSnapshotBody, _ = json.Marshal(gin.H{"Name": snapshotBullseye})
+		resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoBullseye), createSnapshotBody)
+		c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye snapshot"))
+
+		// Publish both snapshots CONCURRENTLY to the SAME prefix
+		// This mimics production where both are published to "external/postgres-auto"
+		// Use the SAME prefix across all iterations to trigger the race more aggressively
+		sharedPrefix := "postgres-auto"
+
+		var wg sync.WaitGroup
+		var trixiePublishCode, bullseyePublishCode int
+
+		wg.Add(2)
+
+		// Publish or update trixie snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "trixie",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false, // Force cleanup to run
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE (this is what happens in production)
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotTrixie},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/trixie", sharedPrefix), updateBody)
+			}
+			trixiePublishCode = resp.Code
+			c.Logf("[iter %d] Trixie publish/update completed: %d", iter, resp.Code)
+		}()
+
+		// Publish or update bullseye snapshot
+		go func() {
+			defer wg.Done()
+
+			var resp *httptest.ResponseRecorder
+			if iter == 0 {
+				// First iteration: CREATE
+				publishBody, _ := json.Marshal(gin.H{
+					"SourceKind":    "snapshot",
+					"Distribution":  "bullseye",
+					"Architectures": []string{"amd64"},
+					"Sources": []gin.H{
+						{"Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody)
+			} else {
+				// Subsequent iterations: UPDATE
+				updateBody, _ := json.Marshal(gin.H{
+					"Snapshots": []gin.H{
+						{"Component": "main", "Name": snapshotBullseye},
+					},
+					"Signing":        gin.H{"Skip": true},
+					"SkipBz2":        true,
+					"ForceOverwrite": true,
+					"SkipCleanup":    false,
+				})
+				resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/bullseye", sharedPrefix), updateBody)
+			}
+			bullseyePublishCode = resp.Code
+			c.Logf("[iter %d] Bullseye publish/update completed: %d", iter, resp.Code)
+		}()
+
+		wg.Wait()
+		time.Sleep(50 * time.Millisecond)
+
+		// Verify publishes succeeded (201 for create, 200 for update)
+		expectedCode := 201
+		if iter > 0 {
+			expectedCode = 200
+		}
+		c.Assert(trixiePublishCode, Equals, expectedCode, Commentf("Trixie publish/update should succeed"))
+		c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed"))
+
+		// Verify ALL package files exist in the published pool
+		publishedStorage, err := s.context.GetPublishedStorage("")
+		c.Assert(err, IsNil)
+		publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath()
+
+		missingFiles := []string{}
+		expectedFiles := []string{}
+
+		// Check trixie packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("TRIXIE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// Check bullseye packages
+		for i := 0; i < numPackages; i++ {
+			packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i)
+			version := fmt.Sprintf("17.0.%d", iter)
+
+			poolSubdir := string(packageName[0])
+			expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName,
+				fmt.Sprintf("%s_%s_amd64.deb", packageName, version))
+
+			expectedFiles = append(expectedFiles, expectedPath)
+			if _, err := os.Stat(expectedPath); os.IsNotExist(err) {
+				missingFiles = append(missingFiles, fmt.Sprintf("BULLSEYE: %s", filepath.Base(expectedPath)))
+			}
+		}
+
+		// BUG: Files from one distribution are deleted by the other's cleanup
+		if len(missingFiles) > 0 {
+			c.Logf("★★★ BUG REPRODUCED in iteration %d/%d! ★★★", iter+1, numIterations)
+			c.Logf("Both publishes to prefix '%s' succeeded, but %d files are MISSING:", sharedPrefix, len(missingFiles))
+			for i, f := range missingFiles {
+				c.Logf("  Missing file %d/%d: %s", i+1, len(missingFiles), f)
+			}
+
+			c.Logf("\nThis reproduces the exact production bug where:")
+			c.Logf("  1. Mirror updates complete successfully")
+			c.Logf("  2. Snapshots are created")
+			c.Logf("  3. Both snapshots publish to same prefix (different distributions)")
+			c.Logf("  4. Cleanup from one publish DELETES files from the other")
+			c.Logf("  5. Result: apt-get returns 404 when downloading packages")
+
+			// List what's actually in the pool
+			poolDir := filepath.Join(publicPath, sharedPrefix, "pool", "main")
+			if entries, err := os.ReadDir(poolDir); err == nil {
+				c.Logf("\nActual pool directory contents (%s):", poolDir)
+				for _, entry := range entries {
+					c.Logf("  - %s/", entry.Name())
+				}
+			}
+
+			c.Fatalf("BUG CONFIRMED (iteration %d/%d): %d files missing from shared pool",
+				iter+1, numIterations, len(missingFiles))
+		} else {
+			c.Logf("[iter %d/%d] All %d files present - OK", iter+1, numIterations, len(expectedFiles))
+		}
+	}
+	c.Logf("✓ All %d iterations passed - no files missing", numIterations)
+}

api/repos.go

@@ -24,7 +24,7 @@ import (
 // @Tags Repos
 // @Produce html
 // @Success 200 {object} string "HTML"
-// @Router /api/repos [get]
+// @Router /repos [get]
 func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
 // @Param pkgPath path string true "Package Path" allowReserved=true
 // @Produce json
 // @Success 200 ""
-// @Router /api/{storage}/{pkgPath} [get]
+// @Router /repos/{storage}/{pkgPath} [get]
 func reposServeInAPIMode(c *gin.Context) {
 	pkgpath := c.Param("pkgPath")
 
@@ -60,7 +60,12 @@ func reposServeInAPIMode(c *gin.Context) {
 		storage = "filesystem:" + storage
 	}
 
-	publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage(storage)
+	if err != nil {
+		AbortWithJSONError(c, http.StatusNotFound, err)
+		return
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	c.FileFromFS(pkgpath, http.Dir(publicPath))
 }
 
@@ -93,7 +98,7 @@ type repoCreateParams struct {
 	DefaultDistribution string `     json:"DefaultDistribution"  example:"stable"`
 	// Default component when publishing from this local repo
 	DefaultComponent string `        json:"DefaultComponent"     example:"main"`
-	// Snapshot name to create repoitory from (optional)
+	// Snapshot name to create repository from (optional)
 	FromSnapshot string `            json:"FromSnapshot"         example:""`
 }
 
@@ -122,63 +127,79 @@ func apiReposCreate(c *gin.Context) {
 		return
 	}
 
-	repo := deb.NewLocalRepo(b.Name, b.Comment)
-	repo.DefaultComponent = b.DefaultComponent
-	repo.DefaultDistribution = b.DefaultDistribution
-
+	// Handler: Pre-task validations (shallow)
 	collectionFactory := context.NewCollectionFactory()
 
+	var resources []string
 	if b.FromSnapshot != "" {
-		var snapshot *deb.Snapshot
-
-		snapshotCollection := collectionFactory.SnapshotCollection()
-
-		snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+		snapshot, err := collectionFactory.SnapshotCollection().ByName(b.FromSnapshot)
 		if err != nil {
 			AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
 			return
 		}
+		resources = append(resources, string(snapshot.Key()))
+	}
 
-		err = snapshotCollection.LoadComplete(snapshot)
-		if err != nil {
-			AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
-			return
+	taskName := fmt.Sprintf("Create repository %s", b.Name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection and check/create ATOMIC inside task
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Check duplicate inside lock
+		if _, err := taskCollection.ByName(b.Name); err == nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+				fmt.Errorf("local repo with name %s already exists", b.Name)
 		}
 
-		repo.UpdateRefList(snapshot.RefList())
-	}
+		// Create repo
+		repo := deb.NewLocalRepo(b.Name, b.Comment)
+		repo.DefaultComponent = b.DefaultComponent
+		repo.DefaultDistribution = b.DefaultDistribution
 
-	localRepoCollection := collectionFactory.LocalRepoCollection()
+		if b.FromSnapshot != "" {
+			snapshotCollection := taskCollectionFactory.SnapshotCollection()
 
-	if _, err := localRepoCollection.ByName(b.Name); err == nil {
-		AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
-		return
-	}
+			snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, 
+					fmt.Errorf("source snapshot not found: %s", err)
+			}
 
-	err := localRepoCollection.Add(repo)
-	if err != nil {
-		AbortWithJSONError(c, http.StatusInternalServerError, err)
-		return
-	}
+			err = snapshotCollection.LoadComplete(snapshot)
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, 
+					fmt.Errorf("unable to load source snapshot: %s", err)
+			}
 
-	c.JSON(http.StatusCreated, repo)
+			repo.UpdateRefList(snapshot.RefList())
+		}
+
+		err := taskCollection.Add(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusCreated, Value: repo}, nil
+	})
 }
 
 type reposEditParams struct {
 	// Name of repository to modify
-	Name *string `binding:"required"     json:"Name"                 example:"repo1"`
+	Name *string `                       json:"Name"                 example:"new-repo-name"`
 	// Change Comment of repository
 	Comment *string `                    json:"Comment"              example:"example repo"`
 	// Change Default Distribution for publishing
 	DefaultDistribution *string `        json:"DefaultDistribution"  example:""`
-	// Change Devault Component for publishing
+	// Change Default Component for publishing
 	DefaultComponent *string `        json:"DefaultComponent"     example:""`
 }
 
 // @Summary Update Repository
 // @Description **Update local repository meta information**
 // @Tags Repos
-// @Param name path string true "Repository name"
+// @Param name path string true "Repository name to modify"
 // @Consume  json
 // @Param request body reposEditParams true "Parameters"
 // @Produce json
@@ -191,42 +212,66 @@ func apiReposEdit(c *gin.Context) {
 	if c.Bind(&b) != nil {
 		return
 	}
-
+	// Load shallowly for 404 check and resource key.
+	// Mutation and duplicate check happen inside the task for atomicity.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
-	if b.Name != nil {
-		_, err := collection.ByName(*b.Name)
-		if err == nil {
-			// already exists
-			AbortWithJSONError(c, 404, err)
+	if b.Name != nil && *b.Name != name {
+		if _, err = collection.ByName(*b.Name); err == nil {
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: local repo %q already exists", *b.Name))
 			return
 		}
-		repo.Name = *b.Name
-	}
-	if b.Comment != nil {
-		repo.Comment = *b.Comment
-	}
-	if b.DefaultDistribution != nil {
-		repo.DefaultDistribution = *b.DefaultDistribution
-	}
-	if b.DefaultComponent != nil {
-		repo.DefaultComponent = *b.DefaultComponent
 	}
 
-	err = collection.Update(repo)
-	if err != nil {
-		AbortWithJSONError(c, 500, err)
-		return
-	}
+	resources := []string{string(repo.Key())}
+	taskName := fmt.Sprintf("Edit repository %s", name)
 
-	c.JSON(200, repo)
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		// Check and update ATOMIC (inside lock)
+		if b.Name != nil && *b.Name != name {
+			_, err := taskCollection.ByName(*b.Name)
+			if err == nil {
+				// already exists
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, 
+					fmt.Errorf("local repo with name %q already exists", *b.Name)
+			}
+			repo.Name = *b.Name
+		}
+		if b.Comment != nil {
+			repo.Comment = *b.Comment
+		}
+		if b.DefaultDistribution != nil {
+			repo.DefaultDistribution = *b.DefaultDistribution
+		}
+		if b.DefaultComponent != nil {
+			repo.DefaultComponent = *b.DefaultComponent
+		}
+
+		err = taskCollection.Update(repo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
+	})
 }
 
 // GET /api/repos/:name
@@ -268,10 +313,10 @@ func apiReposDrop(c *gin.Context) {
 	force := c.Request.URL.Query().Get("force") == "1"
 	name := c.Params.ByName("name")
 
+	// Load shallowly for 404 check, resource key, and task name.
+	// Full checks (published/snapshots) happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	repo, err := collection.ByName(name)
 	if err != nil {
@@ -282,19 +327,32 @@ func apiReposDrop(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Delete repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.ByLocalRepo(repo)
+		// Task: Create fresh collections inside task after lock acquired
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Re-read repo with fresh collection after lock
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: %s", err)
+		}
+
+		// Check with fresh collections
+		published := taskPublishedCollection.ByLocalRepo(repo)
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.ByLocalRepoSource(repo)
+			snapshots := taskSnapshotCollection.ByLocalRepoSource(repo)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
 			}
 		}
 
-		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
+		return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, taskCollection.Drop(repo)
 	})
 }
 
@@ -351,10 +409,13 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
-	repo, err := collection.ByName(c.Params.ByName("name"))
+	name := c.Params.ByName("name")
+	repo, err := collection.ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
@@ -363,13 +424,23 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 	resources := []string{string(repo.Key())}
 
 	maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired (use captured `name` variable, not gin context)
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		out.Printf("Loading packages...\n")
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err := deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -378,7 +449,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 		for _, ref := range b.PackageRefs {
 			var p *deb.Package
 
-			p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
+			p, err = taskCollectionFactory.PackageCollection().ByKey([]byte(ref))
 			if err != nil {
 				if err == database.ErrNotFound {
 					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
@@ -394,7 +465,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -410,6 +481,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
 // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository.
 // @Tags Repos
 // @Param name path string true "Repository name"
+// @Consume  json
 // @Param request body reposPackagesAddDeleteParams true "Parameters"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
@@ -455,7 +527,7 @@ func apiReposPackagesDelete(c *gin.Context) {
 // @Tags Repos
 // @Param name path string true "Repository name"
 // @Param dir path string true "Directory of packages"
-// @Param file path string false "Filename (optional)"
+// @Param file path string true "Filename"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 200 {string} string "OK"
@@ -500,6 +572,8 @@ func apiReposPackageFromDir(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource key.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.LocalRepoCollection()
 
@@ -523,7 +597,17 @@ func apiReposPackageFromDir(c *gin.Context) {
 	resources := []string{string(repo.Key())}
 	resources = append(resources, sources...)
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collection.LoadComplete(repo)
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.LocalRepoCollection()
+
+		// Fresh load after lock acquired
+		repo, err := taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -544,13 +628,13 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
 
-		list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
+		list, err = deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
 		}
 
 		processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
-			collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
+			taskCollectionFactory.PackageCollection(), reporter, nil, taskCollectionFactory.ChecksumCollection)
 		failedFiles = append(failedFiles, failedFiles2...)
 		processedFiles = append(processedFiles, otherFiles...)
 
@@ -560,7 +644,7 @@ func apiReposPackageFromDir(c *gin.Context) {
 
 		repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
 
-		err = collectionFactory.LocalRepoCollection().Update(repo)
+		err = taskCollection.Update(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 		}
@@ -613,11 +697,11 @@ type reposCopyPackageParams struct {
 // @Summary Copy Package
 // @Description Copies a package from a source to destination repository
 // @Tags Repos
-// @Produce json
 // @Param name path string true "Destination repo"
 // @Param src path string true "Source repo"
 // @Param file path string true "File/packages to copy"
 // @Param _async query bool false "Run in background and return task object"
+// @Produce json
 // @Success 200 {object} task.ProcessReturnValue "msg"
 // @Failure 400 {object} Error "Bad Request"
 // @Failure 404 {object} Error "Not Found"
@@ -639,6 +723,8 @@ func apiReposCopyPackage(c *gin.Context) {
 		return
 	}
 
+	// Load shallowly for 404 check and resource keys.
+	// Full load and mutations happen inside the task.
 	collectionFactory := context.NewCollectionFactory()
 	dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
 	if err != nil {
@@ -662,12 +748,26 @@ func apiReposCopyPackage(c *gin.Context) {
 	resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
 
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		// Task: Create fresh factory and collections inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of both repos after lock acquired
+		dstRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(dstRepoName)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
 		}
 
-		err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
+		srcRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(srcRepoName)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
+		}
+
+		err = taskCollectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
 		}
@@ -680,12 +780,12 @@ func apiReposCopyPackage(c *gin.Context) {
 			RemovedLines: []string{},
 		}
 
-		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
 		}
 
-		srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
+		srcList, err := deb.NewPackageListFromRefList(srcRefList, taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
 		}
@@ -753,7 +853,7 @@ func apiReposCopyPackage(c *gin.Context) {
 		} else {
 			dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
 
-			err = collectionFactory.LocalRepoCollection().Update(dstRepo)
+			err = taskCollectionFactory.LocalRepoCollection().Update(dstRepo)
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
 			}
@@ -856,6 +956,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 	resources = append(resources, sources...)
 
 	maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Task: Create fresh factory and collection inside task after lock
+		taskCollectionFactory := context.NewCollectionFactory()
+
 		var (
 			err                       error
 			verifier                  = context.GetVerifier()
@@ -871,8 +974,8 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
 		changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
 		_, failedFiles2, err = deb.ImportChangesFiles(
 			changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
-			repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
-			context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
+			repoTemplate, context.Progress(), taskCollectionFactory.LocalRepoCollection(), taskCollectionFactory.PackageCollection(),
+			context.PackagePool(), taskCollectionFactory.ChecksumCollection, nil, query.Parse)
 		failedFiles = append(failedFiles, failedFiles2...)
 
 		if err != nil {

api/repos_test.go

@@ -1,591 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-
-	"github.com/aptly-dev/aptly/deb"
-	"github.com/aptly-dev/aptly/utils"
-	"github.com/gin-gonic/gin"
-	. "gopkg.in/check.v1"
-)
-
-type ReposTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&ReposTestSuite{})
-
-func (s *ReposTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *ReposTestSuite) TestReposListEmpty(c *C) {
-	// Test listing repos when none exist
-	req, _ := http.NewRequest("GET", "/api/repos", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	var result []*deb.LocalRepo
-	err := json.Unmarshal(w.Body.Bytes(), &result)
-	c.Check(err, IsNil)
-	c.Check(len(result), Equals, 0)
-}
-
-func (s *ReposTestSuite) TestReposCreateBasic(c *C) {
-	// Test creating a basic repository
-	params := repoCreateParams{
-		Name:                "test-repo",
-		Comment:             "Test repository",
-		DefaultDistribution: "stable",
-		DefaultComponent:    "main",
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Now context is properly set up, should create successfully
-	c.Check(w.Code, Equals, 201) // Expect successful creation
-	
-	// Clean up: delete the created repo
-	req, _ = http.NewRequest("DELETE", "/api/repos/test-repo?force=1", nil)
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *ReposTestSuite) TestReposEdit(c *C) {
-	// First create a repo
-	params := repoCreateParams{
-		Name:    "edit-test-repo",
-		Comment: "Original comment",
-	}
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 201)
-
-	// Now edit it
-	editParams := reposEditParams{
-		Comment: stringPtr("Updated comment"),
-	}
-	body, _ = json.Marshal(editParams)
-	req, _ = http.NewRequest("PUT", "/api/repos/edit-test-repo", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 200)
-
-	// Clean up
-	req, _ = http.NewRequest("DELETE", "/api/repos/edit-test-repo?force=1", nil)
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesAddDelete(c *C) {
-	// First create a repo
-	params := repoCreateParams{
-		Name: "pkg-test-repo",
-	}
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 201)
-
-	// Test adding packages (will fail without actual packages)
-	addParams := reposPackagesAddDeleteParams{
-		PackageRefs: []string{"Pamd64 test 1.0 abc123"},
-	}
-	body, _ = json.Marshal(addParams)
-	req, _ = http.NewRequest("POST", "/api/repos/pkg-test-repo/packages", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	// Will fail as package doesn't exist
-	c.Check(w.Code, Not(Equals), 200)
-
-	// Clean up
-	req, _ = http.NewRequest("DELETE", "/api/repos/pkg-test-repo?force=1", nil)
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 200)
-}
-
-func (s *ReposTestSuite) TestReposCopyPackage(c *C) {
-	// Create source and destination repos
-	params := repoCreateParams{Name: "src-repo"}
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 201)
-
-	params = repoCreateParams{Name: "dst-repo"}
-	body, _ = json.Marshal(params)
-	req, _ = http.NewRequest("POST", "/api/repos", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 201)
-
-	// Test copy (will fail without packages)
-	copyParams := reposCopyPackageParams{
-		WithDeps: true,
-		DryRun:   true,
-	}
-	body, _ = json.Marshal(copyParams)
-	req, _ = http.NewRequest("POST", "/api/repos/dst-repo/copy/src-repo/test", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	// Will return empty result as no packages match
-	c.Check(w.Code, Equals, 200)
-
-	// Clean up
-	req, _ = http.NewRequest("DELETE", "/api/repos/src-repo?force=1", nil)
-	s.router.ServeHTTP(w, req)
-	req, _ = http.NewRequest("DELETE", "/api/repos/dst-repo?force=1", nil)
-	s.router.ServeHTTP(w, req)
-}
-
-func (s *ReposTestSuite) TestReposCreateInvalidJSON(c *C) {
-	// Test creating repository with invalid JSON
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewBufferString("invalid json"))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *ReposTestSuite) TestReposCreateMissingName(c *C) {
-	// Test creating repository without required name
-	params := repoCreateParams{
-		Comment:             "Test repository",
-		DefaultDistribution: "stable",
-		DefaultComponent:    "main",
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *ReposTestSuite) TestReposShowNotFound(c *C) {
-	// Test showing non-existent repository
-	req, _ := http.NewRequest("GET", "/api/repos/nonexistent", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests endpoint structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposEditStructure(c *C) {
-	// Test repository edit endpoint structure
-	params := reposEditParams{
-		Name:    stringPtr("new-name"),
-		Comment: stringPtr("Updated comment"),
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("PUT", "/api/repos/test-repo", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposEditInvalidJSON(c *C) {
-	// Test edit with invalid JSON
-	req, _ := http.NewRequest("PUT", "/api/repos/test-repo", bytes.NewBufferString("invalid"))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *ReposTestSuite) TestReposDropStructure(c *C) {
-	// Test repository drop endpoint structure
-	req, _ := http.NewRequest("DELETE", "/api/repos/test-repo", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should return 404 as test-repo doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *ReposTestSuite) TestReposDropWithForce(c *C) {
-	// Test repository drop with force parameter
-	req, _ := http.NewRequest("DELETE", "/api/repos/test-repo?force=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesShowStructure(c *C) {
-	// Test packages show endpoint structure
-	req, _ := http.NewRequest("GET", "/api/repos/test-repo/packages", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesShowWithQuery(c *C) {
-	// Test packages show with query parameters
-	req, _ := http.NewRequest("GET", "/api/repos/test-repo/packages?q=Name%20(~%20test)", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests query parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesAddStructure(c *C) {
-	// Test packages add endpoint structure
-	params := reposPackagesAddDeleteParams{
-		PackageRefs: []string{"Pamd64 test 1.0 abc123"},
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/packages", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesAddInvalidJSON(c *C) {
-	// Test packages add with invalid JSON
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/packages", bytes.NewBufferString("invalid"))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *ReposTestSuite) TestReposPackagesDeleteStructure(c *C) {
-	// Test packages delete endpoint structure
-	params := reposPackagesAddDeleteParams{
-		PackageRefs: []string{"Pamd64 test 1.0 abc123"},
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("DELETE", "/api/repos/test-repo/packages", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposFileUploadStructure(c *C) {
-	// Test file upload endpoint structure
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/file/upload-dir", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposFileUploadWithParameters(c *C) {
-	// Test file upload with query parameters
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/file/upload-dir?noRemove=1&forceReplace=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposFileUploadSpecificFile(c *C) {
-	// Test specific file upload endpoint
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/file/upload-dir/package.deb", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposCopyPackageStructure(c *C) {
-	// Test copy package endpoint structure
-	params := reposCopyPackageParams{
-		WithDeps: true,
-		DryRun:   false,
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos/dest-repo/copy/src-repo/package-query", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposCopyPackageInvalidJSON(c *C) {
-	// Test copy package with invalid JSON
-	req, _ := http.NewRequest("POST", "/api/repos/dest-repo/copy/src-repo/package-query", bytes.NewBufferString("invalid"))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400)
-}
-
-func (s *ReposTestSuite) TestReposIncludePackageStructure(c *C) {
-	// Test include package endpoint structure
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/include/upload-dir", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposIncludePackageWithParameters(c *C) {
-	// Test include package with query parameters
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/include/upload-dir?forceReplace=1&noRemoveFiles=1&acceptUnsigned=1&ignoreSignature=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposIncludeSpecificFile(c *C) {
-	// Test include specific file endpoint
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/include/upload-dir/package.changes", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposParameterValidation(c *C) {
-	// Test parameter validation and structure
-	testCases := []struct {
-		name     string
-		method   string
-		path     string
-		body     string
-		wantCode int
-	}{
-		{"invalid repo name chars", "GET", "/api/repos/invalid/name", "", 404}, // route doesn't match
-		{"empty repo name", "GET", "/api/repos", "", 200}, // list repos endpoint
-		{"invalid method", "PATCH", "/api/repos/test", "", 404},
-		{"malformed JSON in create", "POST", "/api/repos", `{"Name":}`, 400},
-		{"malformed JSON in edit", "PUT", "/api/repos/test", `{"Name":}`, 400},
-		{"malformed JSON in packages", "POST", "/api/repos/test/packages", `{"PackageRefs":}`, 400},
-	}
-
-	for _, tc := range testCases {
-		var req *http.Request
-		if tc.body != "" {
-			req, _ = http.NewRequest(tc.method, tc.path, strings.NewReader(tc.body))
-			req.Header.Set("Content-Type", "application/json")
-		} else {
-			req, _ = http.NewRequest(tc.method, tc.path, nil)
-		}
-
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, tc.wantCode, Commentf("Test case: %s", tc.name))
-	}
-}
-
-func (s *ReposTestSuite) TestReposListInAPIModeStructure(c *C) {
-	// Test reposListInAPIMode function structure
-	localRepos := map[string]utils.FileSystemPublishRoot{
-		"repo1": {},
-		"repo2": {},
-	}
-
-	handler := reposListInAPIMode(localRepos)
-	c.Check(handler, NotNil)
-
-	// Test with empty repos map
-	emptyHandler := reposListInAPIMode(map[string]utils.FileSystemPublishRoot{})
-	c.Check(emptyHandler, NotNil)
-}
-
-func (s *ReposTestSuite) TestReposServeInAPIModeStructure(c *C) {
-	// Test reposServeInAPIMode function structure by simulating call
-	s.router.(*gin.Engine).GET("/api/:storage/*pkgPath", reposServeInAPIMode)
-
-	// Test with default storage
-	req, _ := http.NewRequest("GET", "/api/-/some/package/path", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-
-	// Test with specific storage
-	req, _ = http.NewRequest("GET", "/api/storage1/some/package/path", nil)
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposCreateFromSnapshot(c *C) {
-	// Test creating repository from snapshot
-	params := repoCreateParams{
-		Name:         "test-repo-from-snapshot",
-		Comment:      "Test repository from snapshot",
-		FromSnapshot: "test-snapshot",
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context/snapshot, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposPackagesAsyncOperations(c *C) {
-	// Test async operations with _async parameter
-	params := reposPackagesAddDeleteParams{
-		PackageRefs: []string{"Pamd64 test 1.0 abc123"},
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/packages?_async=1", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests async parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposDropAsyncOperation(c *C) {
-	// Test async repository drop
-	req, _ := http.NewRequest("DELETE", "/api/repos/test-repo?_async=1&force=1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests async parameter parsing
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *ReposTestSuite) TestReposCopyAsyncOperation(c *C) {
-	// Test async copy operation
-	params := reposCopyPackageParams{
-		WithDeps: false,
-		DryRun:   true,
-	}
-
-	jsonBody, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos/dest-repo/copy/src-repo/package-query?_async=1", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-// Helper function to create string pointer
-func stringPtr(s string) *string {
-	return &s
-}
-
-func (s *ReposTestSuite) TestReposPathSanitization(c *C) {
-	// Test path sanitization in file operations
-	testPaths := []string{
-		"../../../etc/passwd",
-		"normal-dir",
-		"dir with spaces",
-		".hidden-dir",
-		"",
-	}
-
-	for _, path := range testPaths {
-		// Test sanitization doesn't cause crashes
-		sanitized := utils.SanitizePath(path)
-		c.Check(sanitized, NotNil)
-
-		// Test with file upload endpoints
-		req, _ := http.NewRequest("POST", fmt.Sprintf("/api/repos/test-repo/file/%s", path), nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not crash, even if it errors due to missing context
-		c.Check(w.Code, Not(Equals), 0)
-	}
-}
-
-func (s *ReposTestSuite) TestReposErrorHandling(c *C) {
-	// Test various error conditions and edge cases
-	errorTests := []struct {
-		description string
-		method      string
-		path        string
-		body        string
-		expectedErr bool
-	}{
-		{"Missing required fields", "POST", "/api/repos", `{}`, true},
-		{"Invalid package refs", "POST", "/api/repos/test/packages", `{"PackageRefs":[]}`, true},
-		{"Invalid query format", "GET", "/api/repos/test/packages?q=invalid[query", "", false}, // Query validation happens deeper
-		{"Copy to same repo", "POST", "/api/repos/test/copy/test/pkg", `{}`, false},            // Error happens in business logic
-		{"File upload endpoint", "POST", "/api/repos/test/file/upload-dir", "", false},         // Valid endpoint
-	}
-
-	for _, test := range errorTests {
-		var req *http.Request
-		if test.body != "" {
-			req, _ = http.NewRequest(test.method, test.path, strings.NewReader(test.body))
-			req.Header.Set("Content-Type", "application/json")
-		} else {
-			req, _ = http.NewRequest(test.method, test.path, nil)
-		}
-
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// All should return some response without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", test.description))
-	}
-}

api/router.go

@@ -11,13 +11,19 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog/log"
 
-	"github.com/aptly-dev/aptly/docs"
-	swaggerFiles "github.com/swaggo/files"
-	ginSwagger "github.com/swaggo/gin-swagger"
+	// _ "github.com/aptly-dev/aptly/docs" // import docs
+	// swaggerFiles "github.com/swaggo/files"
+	// ginSwagger "github.com/swaggo/gin-swagger"
 )
 
 var context *ctx.AptlyContext
 
+// @Summary Get Metrics
+// @Description **Get Prometheus Metrics**
+// @Tags Status
+// @Produce text/plain
+// @Success 200 {string} string Metrics
+// @Router /api/metrics [get]
 func apiMetricsGet() gin.HandlerFunc {
 	return func(c *gin.Context) {
 		countPackagesByRepos()
@@ -25,21 +31,21 @@ func apiMetricsGet() gin.HandlerFunc {
 	}
 }
 
-func redirectSwagger(c *gin.Context) {
-	if c.Request.URL.Path == "/docs/index.html" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs/" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	if c.Request.URL.Path == "/docs" {
-		c.Redirect(http.StatusMovedPermanently, "/docs.html")
-		return
-	}
-	c.Next()
-}
+// func redirectSwagger(c *gin.Context) {
+// 	if c.Request.URL.Path == "/docs/index.html" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs/" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	if c.Request.URL.Path == "/docs" {
+// 		c.Redirect(http.StatusMovedPermanently, "/docs.html")
+// 		return
+// 	}
+// 	c.Next()
+// }
 
 // Router returns prebuilt with routes http.Handler
 func Router(c *ctx.AptlyContext) http.Handler {
@@ -63,21 +69,21 @@ func Router(c *ctx.AptlyContext) http.Handler {
 
 	router.Use(gin.Recovery(), gin.ErrorLogger())
 
-	if c.Config().EnableSwaggerEndpoint {
-		router.GET("docs.html", func(c *gin.Context) {
-			c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
-		})
-		router.Use(redirectSwagger)
-		url := ginSwagger.URL("/docs/doc.json")
-		router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
-	}
+	// if c.Config().EnableSwaggerEndpoint {
+	// 	router.GET("docs.html", func(c *gin.Context) {
+	// 		c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
+	// 	})
+	// 	router.Use(redirectSwagger)
+	// 	url := ginSwagger.URL("/docs/doc.json")
+	// 	router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
+	// }
 
 	if c.Config().EnableMetricsEndpoint {
 		MetricsCollectorRegistrar.Register(router)
 	}
 
 	if c.Config().ServeInAPIMode {
-		router.GET("/repos/", reposListInAPIMode(c.Config().GetFileSystemPublishRoots()))
+		router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
 		router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
 	}
 
@@ -86,17 +92,25 @@ func Router(c *ctx.AptlyContext) http.Handler {
 		// We use a goroutine to count the number of
 		// concurrent requests. When no more requests are
 		// running, we close the database to free the lock.
-		initDBRequests()
+		dbRequests = make(chan dbRequest)
+
+		go acquireDatabase()
 
 		api.Use(func(c *gin.Context) {
-			err := acquireDatabaseConnection()
+			var err error
+
+			errCh := make(chan error)
+			dbRequests <- dbRequest{acquiredb, errCh}
+
+			err = <-errCh
 			if err != nil {
 				AbortWithJSONError(c, 500, err)
 				return
 			}
 
 			defer func() {
-				err := releaseDatabaseConnection()
+				dbRequests <- dbRequest{releasedb, errCh}
+				err = <-errCh
 				if err != nil {
 					AbortWithJSONError(c, 500, err)
 				}

api/router_test.go

@@ -1,18 +0,0 @@
-package api
-
-import (
-	. "gopkg.in/check.v1"
-)
-
-type RouterSuite struct {
-	APISuite
-}
-
-var _ = Suite(&RouterSuite{})
-
-func (s *RouterSuite) TestRedirectSwagger(c *C) {
-	// Test redirect from /docs to /docs/index.html
-	response, _ := s.HTTPRequest("GET", "/docs", nil)
-	c.Check(response.Code, Equals, 301)
-	c.Check(response.Header().Get("Location"), Equals, "/docs/")
-}

api/s3.go

@@ -14,8 +14,7 @@ import (
 // @Router /api/s3 [get]
 func apiS3List(c *gin.Context) {
 	keys := []string{}
-	// Use safe accessor to get a copy of the map
-	s3Roots := context.Config().GetS3PublishRoots()
+	s3Roots := context.Config().S3PublishRoots
 	for k := range s3Roots {
 		keys = append(keys, k)
 	}

api/s3_test.go

@@ -1,18 +0,0 @@
-package api
-
-import (
-	. "gopkg.in/check.v1"
-)
-
-type S3Suite struct {
-	APISuite
-}
-
-var _ = Suite(&S3Suite{})
-
-func (s *S3Suite) TestS3List(c *C) {
-	// Test listing S3 endpoints
-	response, _ := s.HTTPRequest("GET", "/api/s3", nil)
-	c.Check(response.Code, Equals, 200)
-	c.Check(response.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-}

api/snapshot.go

@@ -74,26 +74,33 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.RemoteRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.RemoteRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := repo.CheckLock()
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskMirrorCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = repo.CheckLock()
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
 		}
 
-		err = collection.LoadComplete(repo)
+		err = taskMirrorCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -107,7 +114,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -156,6 +163,7 @@ func apiSnapshotsCreate(c *gin.Context) {
 		}
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 	var resources []string
@@ -169,37 +177,62 @@ func apiSnapshotsCreate(c *gin.Context) {
 			return
 		}
 
-		resources = append(resources, string(sources[i].ResourceKey()))
+		resources = append(resources, string(sources[i].Key()))
 	}
 
 	maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		for i := range sources {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPackageCollection := taskCollectionFactory.PackageCollection()
+
+		// Fresh load of all sources after lock acquired
+		freshSources := make([]*deb.Snapshot, len(b.SourceSnapshots))
+		for i := range b.SourceSnapshots {
+			freshSources[i], err = taskSnapshotCollection.ByName(b.SourceSnapshots[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
 		}
 
-		list := deb.NewPackageList()
+		// Merge packages from all source snapshots
+		var refList *deb.PackageRefList
+		if len(freshSources) > 0 {
+			refList = freshSources[0].RefList()
+			for i := 1; i < len(freshSources); i++ {
+				refList = refList.Merge(freshSources[i].RefList(), true, false)
+			}
+		} else {
+			refList = deb.NewPackageRefList()
+		}
 
-		// verify package refs and build package list
-		for _, ref := range b.PackageRefs {
-			p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
-			if err != nil {
-				if err == database.ErrNotFound {
-					return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+		// Add any explicitly specified package refs on top
+		if len(b.PackageRefs) > 0 {
+			list := deb.NewPackageList()
+			for _, ref := range b.PackageRefs {
+				p, err := taskPackageCollection.ByKey([]byte(ref))
+				if err != nil {
+					if err == database.ErrNotFound {
+						return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
+					}
+					return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+				}
+				err = list.Add(p)
+				if err != nil {
+					return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 				}
-				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-			}
-			err = list.Add(p)
-			if err != nil {
-				return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 			}
+			refList = refList.Merge(deb.NewPackageRefListFromPackageList(list), true, false)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
+		snapshot = deb.NewSnapshotFromRefList(b.Name, freshSources, refList, b.Description)
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -217,10 +250,9 @@ type snapshotsCreateFromRepositoryParams struct {
 // @Summary Snapshot Repository
 // @Description **Create a snapshot of a repository by name**
 // @Tags Snapshots
-// @Param name path string true "Repository name"
 // @Consume json
 // @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
-// @Param name path string true "Name of the snapshot"
+// @Param name path string true "Repository name"
 // @Param _async query bool false "Run in background and return task object"
 // @Produce json
 // @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -241,21 +273,28 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 	}
 
 	collectionFactory := context.NewCollectionFactory()
-	collection := collectionFactory.LocalRepoCollection()
-	snapshotCollection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
 
-	repo, err = collection.ByName(name)
+	repo, err = collectionFactory.LocalRepoCollection().ByName(name)
 	if err != nil {
 		AbortWithJSONError(c, 404, err)
 		return
 	}
 
 	// including snapshot resource key
-	resources := []string{string(repo.Key()), "S" + b.Name}
+	resources := []string{string(repo.Key())}
 	taskName := fmt.Sprintf("Create snapshot of repo %s", name)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err := collection.LoadComplete(repo)
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskRepoCollection := taskCollectionFactory.LocalRepoCollection()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		repo, err := taskRepoCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		err = taskRepoCollection.LoadComplete(repo)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -269,7 +308,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = snapshotCollection.Add(snapshot)
+		err = taskSnapshotCollection.Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
 		}
@@ -307,6 +346,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	collection := collectionFactory.SnapshotCollection()
 	name := c.Params.ByName("name")
@@ -317,14 +357,38 @@ func apiSnapshotsUpdate(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
-	taskName := fmt.Sprintf("Update snapshot %s", name)
-	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		_, err := collection.ByName(b.Name)
+	// Pre-task validation of new name if provided (skip if renaming to same name)
+	if b.Name != "" && b.Name != name {
+		_, err = collection.ByName(b.Name)
 		if err == nil {
-			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
+			return
+		}
+	}
+
+	resources := []string{string(snapshot.Key())}
+	taskName := fmt.Sprintf("Update snapshot %s", name)
+
+	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err = taskCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
+		// Fresh duplicate check inside lock
+		if b.Name != "" {
+			_, err := taskCollection.ByName(b.Name)
+			if err == nil {
+				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
+			}
+		}
+
+		// Update fresh copy
 		if b.Name != "" {
 			snapshot.Name = b.Name
 		}
@@ -333,7 +397,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
 			snapshot.Description = b.Description
 		}
 
-		err = collectionFactory.SnapshotCollection().Update(snapshot)
+		err = taskCollection.Update(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -387,9 +451,9 @@ func apiSnapshotsDrop(c *gin.Context) {
 	name := c.Params.ByName("name")
 	force := c.Request.URL.Query().Get("force") == "1"
 
+	// Phase 1: Pre-task validation (shallow load for 404 check only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
-	publishedCollection := collectionFactory.PublishedRepoCollection()
 
 	snapshot, err := snapshotCollection.ByName(name)
 	if err != nil {
@@ -397,23 +461,37 @@ func apiSnapshotsDrop(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(snapshot.ResourceKey())}
+	resources := []string{string(snapshot.Key())}
 	taskName := fmt.Sprintf("Delete snapshot %s", name)
+
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		published := publishedCollection.BySnapshot(snapshot)
+		// Phase 2: Inside task lock - create fresh collections
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+		taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
+
+		// Fresh load after lock acquired
+		snapshot, err := taskSnapshotCollection.ByName(name)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+
+		// Fresh checks with current collections
+		published := taskPublishedCollection.BySnapshot(snapshot)
 
 		if len(published) > 0 {
 			return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
 		}
 
 		if !force {
-			snapshots := snapshotCollection.BySnapshotSource(snapshot)
+			// Using fresh collection for dependency check
+			snapshots := taskSnapshotCollection.BySnapshotSource(snapshot)
 			if len(snapshots) > 0 {
 				return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
 			}
 		}
 
-		err = snapshotCollection.Drop(snapshot)
+		err = taskSnapshotCollection.Drop(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -568,6 +646,7 @@ func apiSnapshotsMerge(c *gin.Context) {
 		return
 	}
 
+	// Phase 1: Pre-task validation (shallow load for 404 checks only)
 	collectionFactory := context.NewCollectionFactory()
 	snapshotCollection := collectionFactory.SnapshotCollection()
 
@@ -580,36 +659,47 @@ func apiSnapshotsMerge(c *gin.Context) {
 			return
 		}
 
-		resources[i] = string(sources[i].ResourceKey())
+		resources[i] = string(sources[i].Key())
 	}
 
 	maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = snapshotCollection.LoadComplete(sources[0])
-		if err != nil {
-			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
-		}
-		result := sources[0].RefList()
-		for i := 1; i < len(sources); i++ {
-			err = snapshotCollection.LoadComplete(sources[i])
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+		taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
+
+		// Fresh load of all sources inside task
+		freshSources := make([]*deb.Snapshot, len(body.Sources))
+		for i := range body.Sources {
+			freshSources[i], err = taskSnapshotCollection.ByName(body.Sources[i])
 			if err != nil {
 				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 			}
-			result = result.Merge(sources[i].RefList(), overrideMatching, false)
+			// LoadComplete on fresh copy
+			err = taskSnapshotCollection.LoadComplete(freshSources[i])
+			if err != nil {
+				return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+			}
+		}
+
+		// Merge using fresh sources
+		result := freshSources[0].RefList()
+		for i := 1; i < len(freshSources); i++ {
+			result = result.Merge(freshSources[i].RefList(), overrideMatching, false)
 		}
 
 		if latest {
 			result.FilterLatestRefs()
 		}
 
-		sourceDescription := make([]string, len(sources))
-		for i, s := range sources {
+		sourceDescription := make([]string, len(freshSources))
+		for i, s := range freshSources {
 			sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
 		}
 
-		snapshot = deb.NewSnapshotFromRefList(name, sources, result,
+		snapshot = deb.NewSnapshotFromRefList(name, freshSources, result,
 			fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(snapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(snapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
 		}
@@ -690,24 +780,32 @@ func apiSnapshotsPull(c *gin.Context) {
 		return
 	}
 
-	resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
+	resources := []string{string(sourceSnapshot.Key()), string(toSnapshot.Key())}
 	taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
 	maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
-		err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
+		// Phase 2: Inside task lock - create fresh factory
+		taskCollectionFactory := context.NewCollectionFactory()
+
+		// Fresh load of snapshots after lock acquired
+		freshToSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(name)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
+		freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source)
+		if err != nil {
+			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
+		}
+		err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshSourceSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
 
 		// convert snapshots to package list
-		toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		toPackageList, err := deb.NewPackageListFromRefList(freshToSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
-		sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
+		sourcePackageList, err := deb.NewPackageListFromRefList(freshSourceSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}
@@ -804,10 +902,10 @@ func apiSnapshotsPull(c *gin.Context) {
 		}
 
 		// Create <destination> snapshot
-		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
-			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
+		destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{freshToSnapshot, freshSourceSnapshot}, toPackageList,
+			fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", freshToSnapshot.Name, freshSourceSnapshot.Name, strings.Join(body.Queries, ", ")))
 
-		err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
+		err = taskCollectionFactory.SnapshotCollection().Add(destinationSnapshot)
 		if err != nil {
 			return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
 		}

api/snapshot_test.go

@@ -1,496 +0,0 @@
-package api
-
-import (
-	"bytes"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-
-	. "gopkg.in/check.v1"
-)
-
-type SnapshotAPITestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&SnapshotAPITestSuite{})
-
-func (s *SnapshotAPITestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotShow(c *C) {
-	// Test showing a specific snapshot
-	req, _ := http.NewRequest("GET", "/api/snapshots/test-snapshot", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the snapshot doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotUpdate(c *C) {
-	// Test updating a snapshot
-	params := struct {
-		Name        string `json:"Name"`
-		Description string `json:"Description"`
-	}{
-		Name:        "updated-snapshot",
-		Description: "Updated description",
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("PUT", "/api/snapshots/test-snapshot", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the snapshot doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotDrop(c *C) {
-	// Test dropping a snapshot
-	req, _ := http.NewRequest("DELETE", "/api/snapshots/test-snapshot", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the snapshot doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotCreateFromRepository(c *C) {
-	// Test creating a snapshot from repository
-	params := struct {
-		Name        string `json:"Name"`
-		Description string `json:"Description"`
-	}{
-		Name:        "new-snapshot",
-		Description: "Test snapshot",
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/repos/test-repo/snapshots", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the repo doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotDiff(c *C) {
-	// Test diffing two snapshots
-	req, _ := http.NewRequest("GET", "/api/snapshots/snap1/diff/snap2", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the snapshots don't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotSearchPackages(c *C) {
-	// Test searching packages in snapshot
-	req, _ := http.NewRequest("GET", "/api/snapshots/test-snapshot/packages?q=Name", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the snapshot doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotMerge(c *C) {
-	// Test merging snapshots
-	params := struct {
-		Destination string   `json:"Destination"`
-		Sources     []string `json:"Sources"`
-	}{
-		Destination: "merged-snapshot",
-		Sources:     []string{"snap1", "snap2"},
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/snapshots/merge", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return error as snapshots don't exist
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotPull(c *C) {
-	// Test pulling packages between snapshots
-	params := struct {
-		Source      string   `json:"Source"`
-		Destination string   `json:"Destination"`
-		Queries     []string `json:"Queries"`
-	}{
-		Source:      "source-snap",
-		Destination: "dest-snap",
-		Queries:     []string{"Name (~ nginx)"},
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/snapshots/pull", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return error as snapshots don't exist
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotCreateFromMirror(c *C) {
-	// Test creating snapshot from mirror
-	params := struct {
-		Name        string `json:"Name"`
-		Description string `json:"Description"`
-	}{
-		Name:        "mirror-snapshot",
-		Description: "Snapshot from mirror",
-	}
-
-	body, _ := json.Marshal(params)
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots", bytes.NewReader(body))
-	req.Header.Set("Content-Type", "application/json")
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will return 404 as the mirror doesn't exist
-	c.Check(w.Code, Equals, 404)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsListGet(c *C) {
-	// Test GET /api/snapshots endpoint
-	req, _ := http.NewRequest("GET", "/api/snapshots", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle the request without crashing (will likely error due to no context)
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsListWithSort(c *C) {
-	// Test GET /api/snapshots with sort parameter
-	req, _ := http.NewRequest("GET", "/api/snapshots?sort=name", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsListWithDifferentSorts(c *C) {
-	// Test various sort methods
-	sortMethods := []string{"name", "time", "created"}
-
-	for _, sortMethod := range sortMethods {
-		req, _ := http.NewRequest("GET", "/api/snapshots?sort="+sortMethod, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Not(Equals), 0, Commentf("Sort method: %s", sortMethod))
-	}
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreatePost(c *C) {
-	// Test POST /api/snapshots endpoint
-	requestBody := snapshotsCreateParams{
-		Name:            "test-snapshot",
-		Description:     "Test snapshot",
-		SourceSnapshots: []string{"source1"},
-		PackageRefs:     []string{},
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle the request without crashing (will likely error due to no context)
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateInvalidJSON(c *C) {
-	// Test POST with invalid JSON
-	req, _ := http.NewRequest("POST", "/api/snapshots", strings.NewReader("invalid json"))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for invalid JSON
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateMissingName(c *C) {
-	// Test POST with missing required name field
-	requestBody := map[string]interface{}{
-		"Description": "Test without name",
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for missing name
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateFromMirrorPost(c *C) {
-	// Test POST /api/mirrors/{name}/snapshots endpoint
-	requestBody := snapshotsCreateFromMirrorParams{
-		Name:        "mirror-snapshot",
-		Description: "Snapshot from mirror",
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle the request without crashing (will likely error due to no context)
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateFromMirrorInvalidJSON(c *C) {
-	// Test POST with invalid JSON for mirror snapshot
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots", strings.NewReader("invalid json"))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for invalid JSON
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateFromMirrorMissingName(c *C) {
-	// Test POST with missing required name field for mirror snapshot
-	requestBody := map[string]interface{}{
-		"Description": "Mirror snapshot without name",
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for missing name
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateWithAsync(c *C) {
-	// Test POST with async parameter
-	requestBody := snapshotsCreateParams{
-		Name:        "async-snapshot",
-		Description: "Async test snapshot",
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/snapshots?_async=true", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateFromMirrorWithAsync(c *C) {
-	// Test POST mirror snapshot with async parameter
-	requestBody := snapshotsCreateFromMirrorParams{
-		Name:        "async-mirror-snapshot",
-		Description: "Async mirror snapshot",
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots?_async=true", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotsCreateParamsStruct(c *C) {
-	// Test snapshotsCreateParams struct
-	params := snapshotsCreateParams{
-		Name:            "test-name",
-		Description:     "test-description",
-		SourceSnapshots: []string{"snap1", "snap2"},
-		PackageRefs:     []string{"ref1", "ref2"},
-	}
-
-	c.Check(params.Name, Equals, "test-name")
-	c.Check(params.Description, Equals, "test-description")
-	c.Check(params.SourceSnapshots, DeepEquals, []string{"snap1", "snap2"})
-	c.Check(params.PackageRefs, DeepEquals, []string{"ref1", "ref2"})
-}
-
-func (s *SnapshotAPITestSuite) TestSnapshotsCreateFromMirrorParamsStruct(c *C) {
-	// Test snapshotsCreateFromMirrorParams struct
-	params := snapshotsCreateFromMirrorParams{
-		Name:        "mirror-test-name",
-		Description: "mirror-test-description",
-	}
-
-	c.Check(params.Name, Equals, "mirror-test-name")
-	c.Check(params.Description, Equals, "mirror-test-description")
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateEmptyRequest(c *C) {
-	// Test POST with empty request body
-	req, _ := http.NewRequest("POST", "/api/snapshots", strings.NewReader(""))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for empty body
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateFromMirrorEmptyRequest(c *C) {
-	// Test POST mirror snapshot with empty request body
-	req, _ := http.NewRequest("POST", "/api/mirrors/test-mirror/snapshots", strings.NewReader(""))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 400) // Should return bad request for empty body
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsListDefaultSort(c *C) {
-	// Test that default sort is applied when no sort parameter provided
-	req, _ := http.NewRequest("GET", "/api/snapshots", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Endpoint should handle default sort without issues
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateComplexPayload(c *C) {
-	// Test POST with complex payload including all fields
-	requestBody := snapshotsCreateParams{
-		Name:            "complex-snapshot",
-		Description:     "Complex test snapshot with multiple sources",
-		SourceSnapshots: []string{"base-snapshot", "updates-snapshot", "security-snapshot"},
-		PackageRefs:     []string{"pkg1_1.0_amd64", "pkg2_2.0_i386", "pkg3_3.0_all"},
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsHTTPMethods(c *C) {
-	// Test that only allowed HTTP methods work
-
-	// Test unsupported methods for snapshots list
-	deniedMethods := []string{"PUT", "DELETE", "PATCH"}
-	for _, method := range deniedMethods {
-		req, _ := http.NewRequest(method, "/api/snapshots", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 404, Commentf("Method %s should not be allowed for snapshots list", method))
-	}
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateSpecialCharacters(c *C) {
-	// Test snapshot creation with special characters in names
-	specialNames := []string{
-		"snapshot-with-dashes",
-		"snapshot_with_underscores",
-		"snapshot.with.dots",
-		"snapshot123",
-		"UPPERCASESNAPSHOT",
-	}
-
-	for _, name := range specialNames {
-		requestBody := snapshotsCreateParams{
-			Name:        name,
-			Description: "Test snapshot with special characters",
-		}
-
-		jsonBody, _ := json.Marshal(requestBody)
-		req, _ := http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-		req.Header.Set("Content-Type", "application/json")
-
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Not(Equals), 0, Commentf("Special name test failed: %s", name))
-	}
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsListEmptyResponse(c *C) {
-	// Test snapshots list when no snapshots exist
-	req, _ := http.NewRequest("GET", "/api/snapshots", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should return some response (likely error due to no context, but shouldn't crash)
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsCreateWithoutContentType(c *C) {
-	// Test POST without Content-Type header
-	requestBody := `{"Name": "test-snapshot"}`
-	req, _ := http.NewRequest("POST", "/api/snapshots", strings.NewReader(requestBody))
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Should handle missing content type
-	c.Check(w.Code, Not(Equals), 0)
-}
-
-func (s *SnapshotAPITestSuite) TestApiSnapshotsParameterEdgeCases(c *C) {
-	// Test edge cases for parameter validation
-
-	// Test with very long name
-	longName := strings.Repeat("a", 1000)
-	requestBody := snapshotsCreateParams{
-		Name: longName,
-	}
-
-	jsonBody, _ := json.Marshal(requestBody)
-	req, _ := http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-
-	// Test with empty arrays
-	emptyArrayBody := snapshotsCreateParams{
-		Name:            "empty-arrays",
-		SourceSnapshots: []string{},
-		PackageRefs:     []string{},
-	}
-
-	jsonBody, _ = json.Marshal(emptyArrayBody)
-	req, _ = http.NewRequest("POST", "/api/snapshots", bytes.NewBuffer(jsonBody))
-	req.Header.Set("Content-Type", "application/json")
-
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Not(Equals), 0)
-}

api/storage_test.go

@@ -1,71 +0,0 @@
-package api
-
-import (
-	"net/http"
-	"net/http/httptest"
-
-	. "gopkg.in/check.v1"
-)
-
-type StorageTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&StorageTestSuite{})
-
-func (s *StorageTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *StorageTestSuite) TestStorageListStructure(c *C) {
-	// Test storage list endpoint structure
-	req, _ := http.NewRequest("GET", "/api/storage", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	// Should return some storage information without error
-}
-
-func (s *StorageTestSuite) TestStorageHTTPMethods(c *C) {
-	// Test that only GET method is allowed
-	deniedMethods := []string{"POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS"}
-
-	for _, method := range deniedMethods {
-		req, _ := http.NewRequest(method, "/api/storage", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 404, Commentf("Method: %s should be denied", method))
-	}
-}
-
-func (s *StorageTestSuite) TestStorageEndpointReliability(c *C) {
-	// Test multiple calls to ensure endpoint is reliable
-	for i := 0; i < 5; i++ {
-		req, _ := http.NewRequest("GET", "/api/storage", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		c.Check(w.Code, Equals, 200, Commentf("Call #%d", i+1))
-		c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-	}
-}
-
-func (s *StorageTestSuite) TestStorageResponseStructure(c *C) {
-	// Test that response structure is consistent
-	req, _ := http.NewRequest("GET", "/api/storage", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-
-	// Should have valid JSON response
-	body := w.Body.String()
-	c.Check(len(body), Not(Equals), 0)
-
-	// Should start with valid JSON structure
-	c.Check(body[0], Equals, byte('{'), Commentf("Response should be JSON object"))
-}

api/task_test.go

@@ -1,449 +0,0 @@
-package api
-
-import (
-	"net/http"
-	"net/http/httptest"
-
-	. "gopkg.in/check.v1"
-)
-
-type TaskTestSuite struct {
-	APISuite
-}
-
-var _ = Suite(&TaskTestSuite{})
-
-func (s *TaskTestSuite) SetUpTest(c *C) {
-	s.APISuite.SetUpTest(c)
-}
-
-func (s *TaskTestSuite) TestTasksListEmpty(c *C) {
-	// Test listing tasks when none exist
-	req, _ := http.NewRequest("GET", "/api/tasks", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	// Will likely return empty array due to no context, but tests structure
-}
-
-func (s *TaskTestSuite) TestTasksClearStructure(c *C) {
-	// Test clearing tasks
-	req, _ := http.NewRequest("POST", "/api/tasks-clear", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	// Should return empty object
-}
-
-func (s *TaskTestSuite) TestTasksWaitStructure(c *C) {
-	// Test waiting for all tasks
-	req, _ := http.NewRequest("GET", "/api/tasks-wait", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	c.Check(w.Code, Equals, 200)
-	c.Check(w.Header().Get("Content-Type"), Equals, "application/json; charset=utf-8")
-
-	// Should return empty object after waiting
-}
-
-func (s *TaskTestSuite) TestTasksWaitForTaskByIDStructure(c *C) {
-	// Test waiting for specific task by ID
-	req, _ := http.NewRequest("GET", "/api/tasks/123/wait", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksWaitForTaskByIDInvalidID(c *C) {
-	// Test waiting for task with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id+"/wait", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("GET", "/api/tasks/-1/wait", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 400, Commentf("ID: -1 should return 400 (not found)"))
-}
-
-func (s *TaskTestSuite) TestTasksShowStructure(c *C) {
-	// Test showing specific task by ID
-	req, _ := http.NewRequest("GET", "/api/tasks/123", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksShowInvalidID(c *C) {
-	// Test showing task with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("GET", "/api/tasks/-1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 404, Commentf("ID: -1 should return 404 (not found)"))
-	
-	// Test very large number separately - causes int overflow
-	req, _ = http.NewRequest("GET", "/api/tasks/999999999999999999999", nil)
-	w = httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 500, Commentf("Very large number should return 500"))
-}
-
-func (s *TaskTestSuite) TestTasksOutputStructure(c *C) {
-	// Test getting task output by ID
-	req, _ := http.NewRequest("GET", "/api/tasks/123/output", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksOutputInvalidID(c *C) {
-	// Test getting task output with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id+"/output", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("GET", "/api/tasks/-1/output", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 404, Commentf("ID: -1 should return 404 (not found)"))
-}
-
-func (s *TaskTestSuite) TestTasksDetailStructure(c *C) {
-	// Test getting task detail by ID
-	req, _ := http.NewRequest("GET", "/api/tasks/123/detail", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksDetailInvalidID(c *C) {
-	// Test getting task detail with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id+"/detail", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("GET", "/api/tasks/-1/detail", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 404, Commentf("ID: -1 should return 404 (not found)"))
-}
-
-func (s *TaskTestSuite) TestTasksReturnValueStructure(c *C) {
-	// Test getting task return value by ID
-	req, _ := http.NewRequest("GET", "/api/tasks/123/return_value", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksReturnValueInvalidID(c *C) {
-	// Test getting task return value with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id+"/return_value", nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("GET", "/api/tasks/-1/return_value", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 404, Commentf("ID: -1 should return 404 (not found)"))
-}
-
-func (s *TaskTestSuite) TestTasksDeleteStructure(c *C) {
-	// Test deleting task by ID
-	req, _ := http.NewRequest("DELETE", "/api/tasks/123", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-
-	// Will error due to no context or invalid task, but tests structure
-	c.Check(w.Code, Not(Equals), 200)
-}
-
-func (s *TaskTestSuite) TestTasksDeleteInvalidID(c *C) {
-	// Test deleting task with invalid ID
-	invalidIDs := []string{"invalid", "abc", "123.45"} // removed empty string as it causes redirect
-
-	for _, id := range invalidIDs {
-		req, _ := http.NewRequest("DELETE", "/api/tasks/"+id, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should return 500 for invalid ID format
-		c.Check(w.Code, Equals, 500, Commentf("ID: %s", id))
-	}
-	
-	// Test negative ID separately - it's a valid int but invalid task ID
-	req, _ := http.NewRequest("DELETE", "/api/tasks/-1", nil)
-	w := httptest.NewRecorder()
-	s.router.ServeHTTP(w, req)
-	c.Check(w.Code, Equals, 400, Commentf("ID: -1 should return 400 (not found)"))
-}
-
-func (s *TaskTestSuite) TestTasksValidIDFormats(c *C) {
-	// Test various valid ID formats
-	validIDs := []string{"0", "1", "123", "999", "2147483647"} // Max int32
-
-	for _, id := range validIDs {
-		// Test show endpoint
-		req, _ := http.NewRequest("GET", "/api/tasks/"+id, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format), might be 404 (not found) or other error
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-
-		// Test wait endpoint
-		req, _ = http.NewRequest("GET", "/api/tasks/"+id+"/wait", nil)
-		w = httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format)
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-
-		// Test output endpoint
-		req, _ = http.NewRequest("GET", "/api/tasks/"+id+"/output", nil)
-		w = httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format)
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-
-		// Test detail endpoint
-		req, _ = http.NewRequest("GET", "/api/tasks/"+id+"/detail", nil)
-		w = httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format)
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-
-		// Test return_value endpoint
-		req, _ = http.NewRequest("GET", "/api/tasks/"+id+"/return_value", nil)
-		w = httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format)
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-
-		// Test delete endpoint
-		req, _ = http.NewRequest("DELETE", "/api/tasks/"+id, nil)
-		w = httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should not be 500 (invalid format)
-		c.Check(w.Code, Not(Equals), 500, Commentf("ID: %s", id))
-	}
-}
-
-func (s *TaskTestSuite) TestTasksParameterEdgeCases(c *C) {
-	// Test edge cases in parameter handling
-	edgeCases := []struct {
-		path        string
-		description string
-	}{
-		{"/api/tasks/0", "zero ID"},
-		{"/api/tasks/1", "single digit ID"},
-		{"/api/tasks/2147483647", "max int32 ID"},
-		{"/api/tasks/00123", "leading zeros"},
-		{"/api/tasks/+123", "positive sign"},
-	}
-
-	for _, tc := range edgeCases {
-		req, _ := http.NewRequest("GET", tc.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should handle edge cases gracefully without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", tc.description))
-	}
-}
-
-func (s *TaskTestSuite) TestTasksHTTPMethods(c *C) {
-	// Test that correct HTTP methods are supported for each endpoint
-	methodTests := []struct {
-		path           string
-		allowedMethods []string
-		deniedMethods  []string
-	}{
-		{"/api/tasks", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks-clear", []string{"POST"}, []string{"GET", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks-wait", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks/123", []string{"GET", "DELETE"}, []string{"POST", "PUT", "PATCH"}},
-		{"/api/tasks/123/wait", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks/123/output", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks/123/detail", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-		{"/api/tasks/123/return_value", []string{"GET"}, []string{"POST", "PUT", "DELETE", "PATCH"}},
-	}
-
-	for _, test := range methodTests {
-		// Test denied methods return 404 (method not allowed for route)
-		for _, method := range test.deniedMethods {
-			req, _ := http.NewRequest(method, test.path, nil)
-			w := httptest.NewRecorder()
-			s.router.ServeHTTP(w, req)
-
-			c.Check(w.Code, Equals, 404, Commentf("Path: %s, Method: %s", test.path, method))
-		}
-
-		// Test allowed methods are handled (may return errors but not method not allowed)
-		for _, method := range test.allowedMethods {
-			req, _ := http.NewRequest(method, test.path, nil)
-			w := httptest.NewRecorder()
-			s.router.ServeHTTP(w, req)
-
-			// Should handle the request (200, 400, 404 for not found are OK)
-			// Just ensure it's not 0 (no response) or 405 (method not allowed)
-			c.Check(w.Code, Not(Equals), 0, Commentf("Path: %s, Method: %s", test.path, method))
-			c.Check(w.Code, Not(Equals), 405, Commentf("Path: %s, Method: %s", test.path, method))
-		}
-	}
-}
-
-func (s *TaskTestSuite) TestTasksContentTypes(c *C) {
-	// Test content type handling for different endpoints
-	contentTypeTests := []struct {
-		path         string
-		method       string
-		expectedType string
-	}{
-		{"/api/tasks", "GET", "application/json"},
-		{"/api/tasks-clear", "POST", "application/json"},
-		{"/api/tasks-wait", "GET", "application/json"},
-		{"/api/tasks/123", "GET", "application/json"},
-		{"/api/tasks/123/wait", "GET", "application/json"},
-		{"/api/tasks/123/output", "GET", ""}, // Text content
-		{"/api/tasks/123/detail", "GET", "application/json"},
-		{"/api/tasks/123/return_value", "GET", "application/json"},
-	}
-
-	for _, test := range contentTypeTests {
-		req, _ := http.NewRequest(test.method, test.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		if test.expectedType != "" {
-			// Check that JSON endpoints return JSON content type
-			contentType := w.Header().Get("Content-Type")
-			c.Check(contentType, Matches, ".*"+test.expectedType+".*",
-				Commentf("Path: %s, Expected: %s, Got: %s", test.path, test.expectedType, contentType))
-		}
-	}
-}
-
-func (s *TaskTestSuite) TestTasksErrorConditions(c *C) {
-	// Test various error conditions
-	errorTests := []struct {
-		description string
-		path        string
-		method      string
-		expectedErr bool
-	}{
-		{"Non-existent task ID", "/api/tasks/999999", "GET", true},
-		{"Non-existent task wait", "/api/tasks/999999/wait", "GET", true},
-		{"Non-existent task output", "/api/tasks/999999/output", "GET", true},
-		{"Non-existent task detail", "/api/tasks/999999/detail", "GET", true},
-		{"Non-existent task return value", "/api/tasks/999999/return_value", "GET", true},
-		{"Non-existent task delete", "/api/tasks/999999", "DELETE", true},
-		{"Tasks list endpoint", "/api/tasks", "GET", true},                    // Valid endpoint
-		{"Extra path segments", "/api/tasks/123/extra/segment", "GET", false}, // Route not matched
-	}
-
-	for _, test := range errorTests {
-		req, _ := http.NewRequest(test.method, test.path, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// All should return some response without crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Test: %s", test.description))
-	}
-}
-
-func (s *TaskTestSuite) TestTasksResourceManagement(c *C) {
-	// Test that endpoints handle resource management correctly
-	endpoints := []string{
-		"/api/tasks",
-		"/api/tasks-clear",
-		"/api/tasks-wait",
-		"/api/tasks/1",
-		"/api/tasks/1/wait",
-		"/api/tasks/1/output",
-		"/api/tasks/1/detail",
-		"/api/tasks/1/return_value",
-	}
-
-	for _, endpoint := range endpoints {
-		method := "GET"
-		if endpoint == "/api/tasks-clear" {
-			method = "POST"
-		}
-
-		req, _ := http.NewRequest(method, endpoint, nil)
-		w := httptest.NewRecorder()
-		s.router.ServeHTTP(w, req)
-
-		// Should complete without hanging or crashing
-		c.Check(w.Code, Not(Equals), 0, Commentf("Endpoint: %s", endpoint))
-
-		// Response should have proper headers
-		c.Check(w.Header(), NotNil, Commentf("Endpoint: %s", endpoint))
-	}
-}

aptly-etcd-queue.conf

@@ -1,31 +0,0 @@
-{
-  "rootDir": "~/.aptly",
-  "downloadConcurrency": 4,
-  "downloadSpeedLimit": 0,
-  "databaseOpenAttempts": 10,
-  "architectures": ["amd64", "i386", "arm64"],
-  "dependencyFollowSuggests": false,
-  "dependencyFollowRecommends": false,
-  "dependencyFollowAllVariants": false,
-  "dependencyFollowSource": false,
-  "gpgDisableSign": false,
-  "gpgDisableVerify": false,
-  "downloadSourcePackages": false,
-  "ppaDistributorID": "ubuntu",
-  "ppaCodename": "",
-  "s3ConcurrentUploads": 4,
-  "s3UploadQueueSize": 1000,
-  "databaseBackend": {
-    "type": "etcd",
-    "url": "localhost:2379",
-    "timeout": "120s",
-    "writeRetries": 3,
-    "writeQueue": {
-      "enabled": true,
-      "queueSize": 1000,
-      "maxWritesPerSec": 100,
-      "batchMaxSize": 50,
-      "batchMaxWaitMs": 10
-    }
-  }
-}

aptly/aptly_test.go

@@ -1,716 +0,0 @@
-package aptly
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"strings"
-	"testing"
-
-	"github.com/aptly-dev/aptly/utils"
-	. "gopkg.in/check.v1"
-)
-
-// Launch gocheck tests
-func Test(t *testing.T) {
-	TestingT(t)
-}
-
-type AptlySuite struct{}
-
-var _ = Suite(&AptlySuite{})
-
-// Mock implementations for testing interfaces
-
-type MockPackagePool struct {
-	verifyFunc       func(string, string, *utils.ChecksumInfo, ChecksumStorage) (string, bool, error)
-	importFunc       func(string, string, *utils.ChecksumInfo, bool, ChecksumStorage) (string, error)
-	legacyPathFunc   func(string, *utils.ChecksumInfo) (string, error)
-	sizeFunc         func(string) (int64, error)
-	openFunc         func(string) (ReadSeekerCloser, error)
-	filepathListFunc func(Progress) ([]string, error)
-	removeFunc       func(string) (int64, error)
-}
-
-func (m *MockPackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, storage ChecksumStorage) (string, bool, error) {
-	if m.verifyFunc != nil {
-		return m.verifyFunc(poolPath, basename, checksums, storage)
-	}
-	return poolPath, true, nil
-}
-
-func (m *MockPackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (string, error) {
-	if m.importFunc != nil {
-		return m.importFunc(srcPath, basename, checksums, move, storage)
-	}
-	return "imported/path/" + basename, nil
-}
-
-func (m *MockPackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
-	if m.legacyPathFunc != nil {
-		return m.legacyPathFunc(filename, checksums)
-	}
-	return "legacy/" + filename, nil
-}
-
-func (m *MockPackagePool) Size(path string) (int64, error) {
-	if m.sizeFunc != nil {
-		return m.sizeFunc(path)
-	}
-	return 1024, nil
-}
-
-func (m *MockPackagePool) Open(path string) (ReadSeekerCloser, error) {
-	if m.openFunc != nil {
-		return m.openFunc(path)
-	}
-	return &MockReadSeekerCloser{content: []byte("mock file content")}, nil
-}
-
-func (m *MockPackagePool) FilepathList(progress Progress) ([]string, error) {
-	if m.filepathListFunc != nil {
-		return m.filepathListFunc(progress)
-	}
-	return []string{"file1.deb", "file2.deb"}, nil
-}
-
-func (m *MockPackagePool) Remove(path string) (int64, error) {
-	if m.removeFunc != nil {
-		return m.removeFunc(path)
-	}
-	return 1024, nil
-}
-
-type MockReadSeekerCloser struct {
-	content []byte
-	pos     int64
-	closed  bool
-}
-
-func (m *MockReadSeekerCloser) Read(p []byte) (int, error) {
-	if m.closed {
-		return 0, errors.New("closed")
-	}
-	if m.pos >= int64(len(m.content)) {
-		return 0, io.EOF
-	}
-	n := copy(p, m.content[m.pos:])
-	m.pos += int64(n)
-	return n, nil
-}
-
-func (m *MockReadSeekerCloser) Seek(offset int64, whence int) (int64, error) {
-	if m.closed {
-		return 0, errors.New("closed")
-	}
-	switch whence {
-	case io.SeekStart:
-		m.pos = offset
-	case io.SeekCurrent:
-		m.pos += offset
-	case io.SeekEnd:
-		m.pos = int64(len(m.content)) + offset
-	}
-	if m.pos < 0 {
-		m.pos = 0
-	}
-	if m.pos > int64(len(m.content)) {
-		m.pos = int64(len(m.content))
-	}
-	return m.pos, nil
-}
-
-func (m *MockReadSeekerCloser) Close() error {
-	m.closed = true
-	return nil
-}
-
-type MockPublishedStorage struct {
-	mkDirFunc        func(string) error
-	putFileFunc      func(string, string) error
-	removeDirsFunc   func(string, Progress) error
-	removeFunc       func(string) error
-	linkFromPoolFunc func(string, string, string, PackagePool, string, utils.ChecksumInfo, bool) error
-	filelistFunc     func(string) ([]string, error)
-	renameFileFunc   func(string, string) error
-	symLinkFunc      func(string, string) error
-	hardLinkFunc     func(string, string) error
-	fileExistsFunc   func(string) (bool, error)
-	readLinkFunc     func(string) (string, error)
-}
-
-func (m *MockPublishedStorage) MkDir(path string) error {
-	if m.mkDirFunc != nil {
-		return m.mkDirFunc(path)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) PutFile(path, sourceFilename string) error {
-	if m.putFileFunc != nil {
-		return m.putFileFunc(path, sourceFilename)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) RemoveDirs(path string, progress Progress) error {
-	if m.removeDirsFunc != nil {
-		return m.removeDirsFunc(path, progress)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) Remove(path string) error {
-	if m.removeFunc != nil {
-		return m.removeFunc(path)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
-	if m.linkFromPoolFunc != nil {
-		return m.linkFromPoolFunc(publishedPrefix, publishedRelPath, fileName, sourcePool, sourcePath, sourceChecksums, force)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) Filelist(prefix string) ([]string, error) {
-	if m.filelistFunc != nil {
-		return m.filelistFunc(prefix)
-	}
-	return []string{"file1", "file2"}, nil
-}
-
-func (m *MockPublishedStorage) RenameFile(oldName, newName string) error {
-	if m.renameFileFunc != nil {
-		return m.renameFileFunc(oldName, newName)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) SymLink(src, dst string) error {
-	if m.symLinkFunc != nil {
-		return m.symLinkFunc(src, dst)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) HardLink(src, dst string) error {
-	if m.hardLinkFunc != nil {
-		return m.hardLinkFunc(src, dst)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) FileExists(path string) (bool, error) {
-	if m.fileExistsFunc != nil {
-		return m.fileExistsFunc(path)
-	}
-	return true, nil
-}
-
-func (m *MockPublishedStorage) ReadLink(path string) (string, error) {
-	if m.readLinkFunc != nil {
-		return m.readLinkFunc(path)
-	}
-	return "target", nil
-}
-
-func (m *MockPublishedStorage) Flush() error {
-	return nil
-}
-
-type MockProgress struct {
-	buffer      bytes.Buffer
-	started     bool
-	barStarted  bool
-	barProgress int
-}
-
-func (m *MockProgress) Write(p []byte) (n int, err error) {
-	return m.buffer.Write(p)
-}
-
-func (m *MockProgress) Start() {
-	m.started = true
-}
-
-func (m *MockProgress) Shutdown() {
-	m.started = false
-}
-
-func (m *MockProgress) Flush() {
-	// Nothing to do in mock
-}
-
-func (m *MockProgress) InitBar(count int64, isBytes bool, barType BarType) {
-	m.barStarted = true
-}
-
-func (m *MockProgress) ShutdownBar() {
-	m.barStarted = false
-}
-
-func (m *MockProgress) AddBar(count int) {
-	m.barProgress += count
-}
-
-func (m *MockProgress) SetBar(count int) {
-	m.barProgress = count
-}
-
-func (m *MockProgress) Printf(msg string, a ...interface{}) {
-	fmt.Fprintf(&m.buffer, msg, a...)
-}
-
-func (m *MockProgress) ColoredPrintf(msg string, a ...interface{}) {
-	// Strip color codes for testing
-	cleanMsg := strings.ReplaceAll(msg, "@r", "")
-	cleanMsg = strings.ReplaceAll(cleanMsg, "@g", "")
-	cleanMsg = strings.ReplaceAll(cleanMsg, "@y", "")
-	cleanMsg = strings.ReplaceAll(cleanMsg, "@!", "")
-	cleanMsg = strings.ReplaceAll(cleanMsg, "@|", "")
-	fmt.Fprintf(&m.buffer, cleanMsg, a...)
-}
-
-func (m *MockProgress) PrintfStdErr(msg string, a ...interface{}) {
-	fmt.Fprintf(&m.buffer, "[STDERR] "+msg, a...)
-}
-
-type MockDownloader struct {
-	downloadFunc             func(context.Context, string, string) error
-	downloadWithChecksumFunc func(context.Context, string, string, *utils.ChecksumInfo, bool) error
-	progress                 Progress
-	getLengthFunc            func(context.Context, string) (int64, error)
-}
-
-func (m *MockDownloader) Download(ctx context.Context, url, destination string) error {
-	if m.downloadFunc != nil {
-		return m.downloadFunc(ctx, url, destination)
-	}
-	return nil
-}
-
-func (m *MockDownloader) DownloadWithChecksum(ctx context.Context, url, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error {
-	if m.downloadWithChecksumFunc != nil {
-		return m.downloadWithChecksumFunc(ctx, url, destination, expected, ignoreMismatch)
-	}
-	return nil
-}
-
-func (m *MockDownloader) GetProgress() Progress {
-	if m.progress != nil {
-		return m.progress
-	}
-	return &MockProgress{}
-}
-
-func (m *MockDownloader) GetLength(ctx context.Context, url string) (int64, error) {
-	if m.getLengthFunc != nil {
-		return m.getLengthFunc(ctx, url)
-	}
-	return 1024, nil
-}
-
-type MockChecksumStorage struct {
-	getFunc    func(string) (*utils.ChecksumInfo, error)
-	updateFunc func(string, *utils.ChecksumInfo) error
-}
-
-func (m *MockChecksumStorage) Get(path string) (*utils.ChecksumInfo, error) {
-	if m.getFunc != nil {
-		return m.getFunc(path)
-	}
-	return &utils.ChecksumInfo{}, nil
-}
-
-func (m *MockChecksumStorage) Update(path string, c *utils.ChecksumInfo) error {
-	if m.updateFunc != nil {
-		return m.updateFunc(path, c)
-	}
-	return nil
-}
-
-// Test interfaces and their basic functionality
-
-func (s *AptlySuite) TestPackagePoolInterface(c *C) {
-	// Test PackagePool interface with mock implementation
-	var pool PackagePool = &MockPackagePool{}
-
-	checksums := &utils.ChecksumInfo{}
-	mockStorage := &MockChecksumStorage{}
-
-	// Test Verify
-	path, exists, err := pool.Verify("test/path", "package.deb", checksums, mockStorage)
-	c.Check(err, IsNil)
-	c.Check(exists, Equals, true)
-	c.Check(path, Equals, "test/path")
-
-	// Test Import
-	importedPath, err := pool.Import("/src/package.deb", "package.deb", checksums, false, mockStorage)
-	c.Check(err, IsNil)
-	c.Check(importedPath, Equals, "imported/path/package.deb")
-
-	// Test LegacyPath
-	legacyPath, err := pool.LegacyPath("package.deb", checksums)
-	c.Check(err, IsNil)
-	c.Check(legacyPath, Equals, "legacy/package.deb")
-
-	// Test Size
-	size, err := pool.Size("test/path")
-	c.Check(err, IsNil)
-	c.Check(size, Equals, int64(1024))
-
-	// Test Open
-	reader, err := pool.Open("test/path")
-	c.Check(err, IsNil)
-	c.Check(reader, NotNil)
-	reader.Close()
-
-	// Test FilepathList
-	mockProgress := &MockProgress{}
-	files, err := pool.FilepathList(mockProgress)
-	c.Check(err, IsNil)
-	c.Check(len(files), Equals, 2)
-	c.Check(files[0], Equals, "file1.deb")
-
-	// Test Remove
-	removedSize, err := pool.Remove("test/path")
-	c.Check(err, IsNil)
-	c.Check(removedSize, Equals, int64(1024))
-}
-
-func (s *AptlySuite) TestPublishedStorageInterface(c *C) {
-	// Test PublishedStorage interface with mock implementation
-	var storage PublishedStorage = &MockPublishedStorage{}
-
-	// Test MkDir
-	err := storage.MkDir("test/dir")
-	c.Check(err, IsNil)
-
-	// Test PutFile
-	err = storage.PutFile("dest/path", "source/file")
-	c.Check(err, IsNil)
-
-	// Test RemoveDirs
-	mockProgress := &MockProgress{}
-	err = storage.RemoveDirs("test/dir", mockProgress)
-	c.Check(err, IsNil)
-
-	// Test Remove
-	err = storage.Remove("test/file")
-	c.Check(err, IsNil)
-
-	// Test LinkFromPool
-	mockPool := &MockPackagePool{}
-	checksums := utils.ChecksumInfo{}
-	err = storage.LinkFromPool("prefix", "rel/path", "file.deb", mockPool, "pool/path", checksums, false)
-	c.Check(err, IsNil)
-
-	// Test Filelist
-	files, err := storage.Filelist("prefix")
-	c.Check(err, IsNil)
-	c.Check(len(files), Equals, 2)
-
-	// Test RenameFile
-	err = storage.RenameFile("old", "new")
-	c.Check(err, IsNil)
-
-	// Test SymLink
-	err = storage.SymLink("src", "dst")
-	c.Check(err, IsNil)
-
-	// Test HardLink
-	err = storage.HardLink("src", "dst")
-	c.Check(err, IsNil)
-
-	// Test FileExists
-	exists, err := storage.FileExists("test/file")
-	c.Check(err, IsNil)
-	c.Check(exists, Equals, true)
-
-	// Test ReadLink
-	target, err := storage.ReadLink("link")
-	c.Check(err, IsNil)
-	c.Check(target, Equals, "target")
-}
-
-func (s *AptlySuite) TestProgressInterface(c *C) {
-	// Test Progress interface with mock implementation
-	var progress Progress = &MockProgress{}
-
-	// Test Start/Shutdown
-	progress.Start()
-	progress.Shutdown()
-
-	// Test Write
-	n, err := progress.Write([]byte("test"))
-	c.Check(err, IsNil)
-	c.Check(n, Equals, 4)
-
-	// Test progress bar functions
-	progress.InitBar(100, false, BarGeneralBuildPackageList)
-	progress.AddBar(10)
-	progress.SetBar(50)
-	progress.ShutdownBar()
-
-	// Test Printf functions
-	progress.Printf("test %s", "message")
-	progress.ColoredPrintf("colored %s", "message")
-	progress.PrintfStdErr("error %s", "message")
-
-	// Test Flush
-	progress.Flush()
-}
-
-func (s *AptlySuite) TestDownloaderInterface(c *C) {
-	// Test Downloader interface with mock implementation
-	var downloader Downloader = &MockDownloader{}
-
-	ctx := context.Background()
-
-	// Test Download
-	err := downloader.Download(ctx, "http://example.com/file", "/tmp/dest")
-	c.Check(err, IsNil)
-
-	// Test DownloadWithChecksum
-	checksums := &utils.ChecksumInfo{}
-	err = downloader.DownloadWithChecksum(ctx, "http://example.com/file", "/tmp/dest", checksums, false)
-	c.Check(err, IsNil)
-
-	// Test GetProgress
-	progress := downloader.GetProgress()
-	c.Check(progress, NotNil)
-
-	// Test GetLength
-	length, err := downloader.GetLength(ctx, "http://example.com/file")
-	c.Check(err, IsNil)
-	c.Check(length, Equals, int64(1024))
-}
-
-func (s *AptlySuite) TestChecksumStorageInterface(c *C) {
-	// Test ChecksumStorage interface with mock implementation
-	var storage ChecksumStorage = &MockChecksumStorage{}
-
-	// Test Get
-	checksums, err := storage.Get("test/path")
-	c.Check(err, IsNil)
-	c.Check(checksums, NotNil)
-
-	// Test Update
-	newChecksums := &utils.ChecksumInfo{}
-	err = storage.Update("test/path", newChecksums)
-	c.Check(err, IsNil)
-}
-
-func (s *AptlySuite) TestConsoleResultReporter(c *C) {
-	// Test ConsoleResultReporter implementation
-	mockProgress := &MockProgress{}
-	reporter := &ConsoleResultReporter{Progress: mockProgress}
-
-	// Test interface compliance
-	var _ ResultReporter = reporter
-
-	// Test Warning
-	reporter.Warning("test warning %s", "message")
-	output := mockProgress.buffer.String()
-	c.Check(strings.Contains(output, "test warning message"), Equals, true)
-	c.Check(strings.Contains(output, "[!]"), Equals, true)
-
-	// Reset buffer
-	mockProgress.buffer.Reset()
-
-	// Test Removed
-	reporter.Removed("removed %s", "item")
-	output = mockProgress.buffer.String()
-	c.Check(strings.Contains(output, "removed item"), Equals, true)
-	c.Check(strings.Contains(output, "[-]"), Equals, true)
-
-	// Reset buffer
-	mockProgress.buffer.Reset()
-
-	// Test Added
-	reporter.Added("added %s", "item")
-	output = mockProgress.buffer.String()
-	c.Check(strings.Contains(output, "added item"), Equals, true)
-	c.Check(strings.Contains(output, "[+]"), Equals, true)
-}
-
-func (s *AptlySuite) TestRecordingResultReporter(c *C) {
-	// Test RecordingResultReporter implementation
-	reporter := &RecordingResultReporter{
-		Warnings:     []string{},
-		AddedLines:   []string{},
-		RemovedLines: []string{},
-	}
-
-	// Test interface compliance
-	var _ ResultReporter = reporter
-
-	// Test Warning
-	reporter.Warning("test warning %s", "message")
-	c.Check(len(reporter.Warnings), Equals, 1)
-	c.Check(reporter.Warnings[0], Equals, "test warning message")
-
-	// Test Removed
-	reporter.Removed("removed %s", "item")
-	c.Check(len(reporter.RemovedLines), Equals, 1)
-	c.Check(reporter.RemovedLines[0], Equals, "removed item")
-
-	// Test Added
-	reporter.Added("added %s", "item")
-	c.Check(len(reporter.AddedLines), Equals, 1)
-	c.Check(reporter.AddedLines[0], Equals, "added item")
-
-	// Test multiple entries
-	reporter.Warning("second warning")
-	reporter.Added("second addition")
-	c.Check(len(reporter.Warnings), Equals, 2)
-	c.Check(len(reporter.AddedLines), Equals, 2)
-	c.Check(reporter.Warnings[1], Equals, "second warning")
-	c.Check(reporter.AddedLines[1], Equals, "second addition")
-}
-
-func (s *AptlySuite) TestReadSeekerCloserInterface(c *C) {
-	// Test ReadSeekerCloser interface with mock implementation
-	var rsc ReadSeekerCloser = &MockReadSeekerCloser{
-		content: []byte("Hello, World!"),
-	}
-
-	// Test Read
-	buf := make([]byte, 5)
-	n, err := rsc.Read(buf)
-	c.Check(err, IsNil)
-	c.Check(n, Equals, 5)
-	c.Check(string(buf), Equals, "Hello")
-
-	// Test Seek
-	pos, err := rsc.Seek(0, io.SeekStart)
-	c.Check(err, IsNil)
-	c.Check(pos, Equals, int64(0))
-
-	// Test Read again from beginning
-	n, err = rsc.Read(buf)
-	c.Check(err, IsNil)
-	c.Check(string(buf), Equals, "Hello")
-
-	// Test Seek to end
-	pos, err = rsc.Seek(-6, io.SeekEnd)
-	c.Check(err, IsNil)
-	c.Check(pos, Equals, int64(7))
-
-	// Test Read from near end
-	buf = make([]byte, 10)
-	n, err = rsc.Read(buf)
-	c.Check(err, IsNil)
-	c.Check(string(buf[:n]), Equals, "World!")
-
-	// Test Close
-	err = rsc.Close()
-	c.Check(err, IsNil)
-
-	// Test Read after close (should error)
-	_, err = rsc.Read(buf)
-	c.Check(err, NotNil)
-}
-
-func (s *AptlySuite) TestBarTypeConstants(c *C) {
-	// Test BarType constants are defined and different
-	barTypes := []BarType{
-		BarGeneralBuildPackageList,
-		BarGeneralVerifyDependencies,
-		BarGeneralBuildFileList,
-		BarCleanupBuildList,
-		BarCleanupDeleteUnreferencedFiles,
-		BarMirrorUpdateDownloadIndexes,
-		BarMirrorUpdateDownloadPackages,
-		BarMirrorUpdateBuildPackageList,
-		BarMirrorUpdateImportFiles,
-		BarMirrorUpdateFinalizeDownload,
-		BarPublishGeneratePackageFiles,
-		BarPublishFinalizeIndexes,
-	}
-
-	// Check that all constants are different
-	seen := make(map[BarType]bool)
-	for _, barType := range barTypes {
-		c.Check(seen[barType], Equals, false, Commentf("Duplicate BarType: %v", barType))
-		seen[barType] = true
-	}
-
-	// Check that they are sequential integers starting from 0
-	for i, barType := range barTypes {
-		c.Check(int(barType), Equals, i, Commentf("BarType not sequential: %v", barType))
-	}
-}
-
-func (s *AptlySuite) TestErrorHandling(c *C) {
-	// Test error handling in mock implementations
-
-	// Test PackagePool with errors
-	pool := &MockPackagePool{
-		verifyFunc: func(string, string, *utils.ChecksumInfo, ChecksumStorage) (string, bool, error) {
-			return "", false, errors.New("verify error")
-		},
-		importFunc: func(string, string, *utils.ChecksumInfo, bool, ChecksumStorage) (string, error) {
-			return "", errors.New("import error")
-		},
-	}
-
-	_, _, err := pool.Verify("", "", nil, nil)
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "verify error")
-
-	_, err = pool.Import("", "", nil, false, nil)
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "import error")
-
-	// Test PublishedStorage with errors
-	storage := &MockPublishedStorage{
-		mkDirFunc: func(string) error {
-			return errors.New("mkdir error")
-		},
-		fileExistsFunc: func(string) (bool, error) {
-			return false, errors.New("file exists error")
-		},
-	}
-
-	err = storage.MkDir("test")
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "mkdir error")
-
-	_, err = storage.FileExists("test")
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "file exists error")
-}
-
-func (s *AptlySuite) TestInterfaceCompatibility(c *C) {
-	// Test that our mocks properly implement the interfaces
-
-	// PackagePool interface
-	var _ PackagePool = &MockPackagePool{}
-
-	// PublishedStorage interface
-	var _ PublishedStorage = &MockPublishedStorage{}
-
-	// Progress interface
-	var _ Progress = &MockProgress{}
-
-	// Downloader interface
-	var _ Downloader = &MockDownloader{}
-
-	// ChecksumStorage interface
-	var _ ChecksumStorage = &MockChecksumStorage{}
-
-	// ReadSeekerCloser interface
-	var _ ReadSeekerCloser = &MockReadSeekerCloser{}
-
-	// ResultReporter interface
-	var _ ResultReporter = &ConsoleResultReporter{}
-	var _ ResultReporter = &RecordingResultReporter{}
-
-	// Test that the interface checks pass
-	c.Check(true, Equals, true)
-}

aptly/interfaces.go

@@ -85,8 +85,6 @@ type PublishedStorage interface {
 	FileExists(path string) (bool, error)
 	// ReadLink returns the symbolic link pointed to by path
 	ReadLink(path string) (string, error)
-	// Flush waits for any pending operations to complete (used by concurrent upload implementations)
-	Flush() error
 }
 
 // FileSystemPublishedStorage is published storage on filesystem
@@ -97,8 +95,8 @@ type FileSystemPublishedStorage interface {
 
 // PublishedStorageProvider is a thing that returns PublishedStorage by name
 type PublishedStorageProvider interface {
-	// GetPublishedStorage returns PublishedStorage by name
-	GetPublishedStorage(name string) PublishedStorage
+	// GetPublishedStorage returns PublishedStorage by name, or an error if the storage is not configured
+	GetPublishedStorage(name string) (PublishedStorage, error)
 }
 
 // BarType used to differentiate between different progress bars

aptly/interfaces_test.go

@@ -1,25 +0,0 @@
-package aptly
-
-import (
-	. "gopkg.in/check.v1"
-)
-
-type InterfacesSuite struct{}
-
-var _ = Suite(&InterfacesSuite{})
-
-func (s *InterfacesSuite) TestBarTypeValues(c *C) {
-	// Test that BarType enum values are as expected
-	c.Check(int(BarGeneralBuildPackageList), Equals, 0)
-	c.Check(int(BarGeneralVerifyDependencies), Equals, 1)
-	c.Check(int(BarGeneralBuildFileList), Equals, 2)
-	c.Check(int(BarCleanupBuildList), Equals, 3)
-	c.Check(int(BarCleanupDeleteUnreferencedFiles), Equals, 4)
-	c.Check(int(BarMirrorUpdateDownloadIndexes), Equals, 5)
-	c.Check(int(BarMirrorUpdateDownloadPackages), Equals, 6)
-	c.Check(int(BarMirrorUpdateBuildPackageList), Equals, 7)
-	c.Check(int(BarMirrorUpdateImportFiles), Equals, 8)
-	c.Check(int(BarMirrorUpdateFinalizeDownload), Equals, 9)
-	c.Check(int(BarPublishGeneratePackageFiles), Equals, 10)
-	c.Check(int(BarPublishFinalizeIndexes), Equals, 11)
-}

azure/azure.go

@@ -5,35 +5,28 @@ package azure
 import (
 	"context"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io"
-	"os"
+	"net/url"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 )
 
 func isBlobNotFound(err error) bool {
-	var respErr *azcore.ResponseError
-	if errors.As(err, &respErr) {
-		return respErr.StatusCode == 404 // BlobNotFound
-	}
-	return false
+	storageError, ok := err.(azblob.StorageError)
+	return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
 }
 
 type azContext struct {
-	client    *azblob.Client
-	container string
+	container azblob.ContainerURL
 	prefix    string
 }
 
 func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
-	cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
+	credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
 	if err != nil {
 		return nil, err
 	}
@@ -42,14 +35,15 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) (
 		endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
 	}
 
-	serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
+	url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
 	if err != nil {
 		return nil, err
 	}
 
+	containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
+
 	result := &azContext{
-		client:    serviceClient,
-		container: container,
+		container: containerURL,
 		prefix:    prefix,
 	}
 
@@ -60,6 +54,10 @@ func (az *azContext) blobPath(path string) string {
 	return filepath.Join(az.prefix, path)
 }
 
+func (az *azContext) blobURL(path string) azblob.BlobURL {
+	return az.container.NewBlobURL(az.blobPath(path))
+}
+
 func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
 	const delimiter = "/"
 	paths = make([]string, 0, 1024)
@@ -69,33 +67,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 		prefix += delimiter
 	}
 
-	ctx := context.Background()
-	maxResults := int32(1)
-	pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
-		Prefix:     &prefix,
-		MaxResults: &maxResults,
-		Include:    azblob.ListBlobsInclude{Metadata: true},
-	})
-
-	// Iterate over each page
-	for pager.More() {
-		page, err := pager.NextPage(ctx)
+	for marker := (azblob.Marker{}); marker.NotDone(); {
+		listBlob, err := az.container.ListBlobsFlatSegment(
+			context.Background(), marker, azblob.ListBlobsSegmentOptions{
+				Prefix:     prefix,
+				MaxResults: 1,
+				Details:    azblob.BlobListingDetails{Metadata: true}})
 		if err != nil {
 			return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
 		}
 
-		for _, blob := range page.Segment.BlobItems {
-			if prefix == "" {
-				paths = append(paths, *blob.Name)
-			} else {
-				name := *blob.Name
-				paths = append(paths, name[len(prefix):])
-			}
-			b := *blob
-			md5 := b.Properties.ContentMD5
-			md5s = append(md5s, fmt.Sprintf("%x", md5))
+		marker = listBlob.NextMarker
 
+		for _, blob := range listBlob.Segment.BlobItems {
+			if prefix == "" {
+				paths = append(paths, blob.Name)
+			} else {
+				paths = append(paths, blob.Name[len(prefix):])
+			}
+			md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
 		}
+
 		if progress != nil {
 			time.Sleep(time.Duration(500) * time.Millisecond)
 			progress.AddBar(1)
@@ -105,27 +97,28 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
 	return paths, md5s, nil
 }
 
-func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
-	uploadOptions := &azblob.UploadFileOptions{
-		BlockSize:   4 * 1024 * 1024,
-		Concurrency: 8,
+func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
+	uploadOptions := azblob.UploadStreamToBlockBlobOptions{
+		BufferSize: 4 * 1024 * 1024,
+		MaxBuffers: 8,
 	}
 
-	path := az.blobPath(blobName)
 	if len(sourceMD5) > 0 {
 		decodedMD5, err := hex.DecodeString(sourceMD5)
 		if err != nil {
 			return err
 		}
-		uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
-			BlobContentMD5: decodedMD5,
+		uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
+			ContentMD5: decodedMD5,
 		}
 	}
 
-	var err error
-	if file, ok := source.(*os.File); ok {
-		_, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
-	}
+	_, err := azblob.UploadStreamToBlockBlob(
+		context.Background(),
+		source,
+		blob.ToBlockBlobURL(),
+		uploadOptions,
+	)
 
 	return err
 }

azure/package_pool.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
 	"github.com/pkg/errors"
@@ -40,7 +41,10 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu
 	return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
 }
 
-func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
+func (pool *PackagePool) ensureChecksums(
+	poolPath string,
+	checksumStorage aptly.ChecksumStorage,
+) (*utils.ChecksumInfo, error) {
 	targetChecksums, err := checksumStorage.Get(poolPath)
 	if err != nil {
 		return nil, err
@@ -48,7 +52,8 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 
 	if targetChecksums == nil {
 		// we don't have checksums stored yet for this file
-		download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
+		blob := pool.az.blobURL(poolPath)
+		download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 		if err != nil {
 			if isBlobNotFound(err) {
 				return nil, nil
@@ -58,7 +63,7 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.
 		}
 
 		targetChecksums = &utils.ChecksumInfo{}
-		*targetChecksums, err = utils.ChecksumsForReader(download.Body)
+		*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
 		if err != nil {
 			return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
 		}
@@ -87,49 +92,45 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er
 }
 
 func (pool *PackagePool) Size(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
+	blob := pool.az.blobURL(path)
+
 	temp, err := os.CreateTemp("", "blob-download")
 	if err != nil {
-		return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
+		return nil, errors.Wrap(err, "error creating temporary file for blob download")
 	}
 	defer func() { _ = os.Remove(temp.Name()) }()
 
-	_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
+	err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
 	if err != nil {
-		return nil, errors.Wrapf(err, "error downloading blob %s", path)
+		return nil, errors.Wrapf(err, "error downloading blob at %s", path)
 	}
 
 	return temp, nil
 }
 
 func (pool *PackagePool) Remove(path string) (int64, error) {
-	serviceClient := pool.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(pool.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-
-	props, err := blobClient.GetProperties(context.TODO(), nil)
+	blob := pool.az.blobURL(path)
+	props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
+		return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
 	}
 
-	_, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
+	_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
 	}
 
-	return *props.ContentLength, nil
+	return props.ContentLength(), nil
 }
 
 func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
@@ -143,6 +144,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	}
 
 	path := pool.buildPoolPath(basename, checksums)
+	blob := pool.az.blobURL(path)
 	targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
 	if err != nil {
 		return "", err
@@ -158,7 +160,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
 	}
 	defer func() { _ = source.Close() }()
 
-	err = pool.az.putFile(path, source, checksums.MD5)
+	err = pool.az.putFile(blob, source, checksums.MD5)
 	if err != nil {
 		return "", err
 	}

azure/package_pool_test.go

@@ -7,7 +7,7 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
@@ -50,10 +50,8 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
 
 	s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-	publicAccessType := azblob.PublicAccessTypeContainer
-	_, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
-		Access: &publicAccessType,
-	})
+	cnt := s.pool.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)

azure/public.go

@@ -3,22 +3,19 @@ package azure
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 )
 
 // PublishedStorage abstract file system with published files (actually hosted on Azure)
 type PublishedStorage struct {
-	// FIXME: unused ???? prefix    string
 	az        *azContext
 	pathCache map[string]map[string]string
 }
@@ -67,7 +64,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 	}
 	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(path, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
 	}
@@ -77,15 +74,14 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
 
 // RemoveDirs removes directory structure under public path
 func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
-	path = storage.az.blobPath(path)
 	filelist, err := storage.Filelist(path)
 	if err != nil {
 		return err
 	}
 
 	for _, filename := range filelist {
-		blob := filepath.Join(path, filename)
-		_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
+		blob := storage.az.blobURL(filepath.Join(path, filename))
+		_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 		if err != nil {
 			return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
 		}
@@ -96,8 +92,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
 
 // Remove removes single file under public path
 func (storage *PublishedStorage) Remove(path string) error {
-	path = storage.az.blobPath(path)
-	_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
+	blob := storage.az.blobURL(path)
+	_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
 		err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
 	}
@@ -116,8 +112,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
 
 	relFilePath := filepath.Join(publishedRelPath, fileName)
-	prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
-	poolPath := storage.az.blobPath(prefixRelFilePath)
+	// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
+	// FIXME: check how to integrate publishedPrefix:
+	poolPath := storage.az.blobPath(fileName)
 
 	if storage.pathCache == nil {
 		storage.pathCache = make(map[string]map[string]string)
@@ -160,7 +157,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
 	}
 	defer func() { _ = source.Close() }()
 
-	err = storage.az.putFile(relFilePath, source, sourceMD5)
+	err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
 	if err == nil {
 		pathCache[relFilePath] = sourceMD5
 	} else {
@@ -177,60 +174,57 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
 }
 
 // Internal copy or move implementation
-func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
+func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
 	const leaseDuration = 30
-	leaseID := uuid.NewString()
 
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	srcBlobClient := containerClient.NewBlobClient(src)
-	blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
+	dstBlobURL := storage.az.blobURL(dst)
+	srcBlobURL := storage.az.blobURL(src)
+	leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
+	if err != nil || leaseResp.StatusCode() != http.StatusCreated {
+		return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
 	}
+	defer func() { _, _ = srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) }()
+	srcBlobLeaseID := leaseResp.LeaseID()
 
-	_, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
-	if err != nil {
-		return fmt.Errorf("error acquiring lease on source blob %s", src)
-	}
-	defer func() {
-		_, _ = blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
-	}()
-
-	dstBlobClient := containerClient.NewBlobClient(dst)
-	copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
-		Metadata: metadata,
-	})
-
+	copyResp, err := dstBlobURL.StartCopyFromURL(
+		context.Background(),
+		srcBlobURL.URL(),
+		metadata,
+		azblob.ModifiedAccessConditions{},
+		azblob.BlobAccessConditions{},
+		azblob.DefaultAccessTier,
+		nil)
 	if err != nil {
 		return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
 	}
 
-	copyStatus := *copyResp.CopyStatus
+	copyStatus := copyResp.CopyStatus()
 	for {
-		if copyStatus == blob.CopyStatusTypeSuccess {
+		if copyStatus == azblob.CopyStatusSuccess {
 			if move {
-				_, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
-					AccessConditions: &blob.AccessConditions{
-						LeaseAccessConditions: &blob.LeaseAccessConditions{
-							LeaseID: &leaseID,
-						},
-					},
-				})
+				_, err = srcBlobURL.Delete(
+					context.Background(),
+					azblob.DeleteSnapshotsOptionNone,
+					azblob.BlobAccessConditions{
+						LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID},
+					})
 				return err
 			}
 			return nil
-		} else if copyStatus == blob.CopyStatusTypePending {
+		} else if copyStatus == azblob.CopyStatusPending {
 			time.Sleep(1 * time.Second)
-			getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
+			blobPropsResp, err := dstBlobURL.GetProperties(
+				context.Background(),
+				azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
+				azblob.ClientProvidedKeyOptions{})
 			if err != nil {
-				return fmt.Errorf("error getting copy progress %s", dst)
+				return fmt.Errorf("error getting destination blob properties %s", dstBlobURL)
 			}
-			copyStatus = *getMetadata.CopyStatus
+			copyStatus = blobPropsResp.CopyStatus()
 
-			_, err = blobLeaseClient.RenewLease(context.Background(), nil)
+			_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{})
 			if err != nil {
-				return fmt.Errorf("error renewing source blob lease %s", src)
+				return fmt.Errorf("error renewing source blob lease %s", srcBlobURL)
 			}
 		} else {
 			return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -245,9 +239,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
 
 // SymLink creates a copy of src file and adds link information as meta data
 func (storage *PublishedStorage) SymLink(src string, dst string) error {
-	metadata := make(map[string]*string)
-	metadata["SymLink"] = &src
-	return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
+	return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */)
 }
 
 // HardLink using symlink functionality as hard links do not exist
@@ -257,38 +249,28 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
 
 // FileExists returns true if path exists
 func (storage *PublishedStorage) FileExists(path string) (bool, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	_, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		if isBlobNotFound(err) {
 			return false, nil
 		}
-		return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
+		return false, err
+	} else if resp.StatusCode() == http.StatusOK {
+		return true, nil
 	}
-	return true, nil
+	return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 }
 
 // ReadLink returns the symbolic link pointed to by path.
 // This simply reads text file created with SymLink
 func (storage *PublishedStorage) ReadLink(path string) (string, error) {
-	serviceClient := storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	props, err := blobClient.GetProperties(context.Background(), nil)
+	blob := storage.az.blobURL(path)
+	resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
-		return "", fmt.Errorf("failed to get blob properties: %v", err)
+		return "", err
+	} else if resp.StatusCode() != http.StatusOK {
+		return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
 	}
-
-	metadata := props.Metadata
-	if originalBlob, exists := metadata["original_blob"]; exists {
-		return *originalBlob, nil
-	}
-	return "", fmt.Errorf("error reading link %s: %v", path, err)
-}
-
-// Flush is a no-op for Azure storage
-func (storage *PublishedStorage) Flush() error {
-	return nil
+	return resp.NewMetadata()["SymLink"], nil
 }

azure/public_test.go

@@ -1,7 +1,6 @@
 package azure
 
 import (
-	"bytes"
 	"context"
 	"crypto/md5"
 	"crypto/rand"
@@ -9,9 +8,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
+	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/aptly-dev/aptly/files"
 	"github.com/aptly-dev/aptly/utils"
 	. "gopkg.in/check.v1"
@@ -69,10 +66,8 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 
 	s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
 	c.Assert(err, IsNil)
-	publicAccessType := azblob.PublicAccessTypeContainer
-	_, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
-		Access: &publicAccessType,
-	})
+	cnt := s.storage.az.container
+	_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
 	c.Assert(err, IsNil)
 
 	s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -80,39 +75,41 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
 }
 
 func (s *PublishedStorageSuite) TearDownTest(c *C) {
-	_, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
+	cnt := s.storage.az.container
+	_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
 	c.Assert(err, IsNil)
 }
 
 func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
-	resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
+	blob := s.storage.az.container.NewBlobURL(path)
+	resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, IsNil)
-	data, err := io.ReadAll(resp.Body)
+	body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
+	data, err := io.ReadAll(body)
 	c.Assert(err, IsNil)
 	return data
 }
 
 func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
-	serviceClient := s.storage.az.client.ServiceClient()
-	containerClient := serviceClient.NewContainerClient(s.storage.az.container)
-	blobClient := containerClient.NewBlobClient(path)
-	_, err := blobClient.GetProperties(context.Background(), nil)
+	_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
+		context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
 	c.Assert(err, NotNil)
-
-	storageError, ok := err.(*azcore.ResponseError)
+	storageError, ok := err.(azblob.StorageError)
 	c.Assert(ok, Equals, true)
-	c.Assert(storageError.StatusCode, Equals, 404)
+	c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound)))
 }
 
 func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
 	hash := md5.Sum(data)
-	uploadOptions := &azblob.UploadStreamOptions{
-		HTTPHeaders: &blob.HTTPHeaders{
-			BlobContentMD5: hash[:],
-		},
-	}
-	reader := bytes.NewReader(data)
-	_, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
+	_, err := azblob.UploadBufferToBlockBlob(
+		context.Background(),
+		data,
+		s.storage.az.container.NewBlockBlobURL(path),
+		azblob.UploadToBlockBlobOptions{
+			BlobHTTPHeaders: azblob.BlobHTTPHeaders{
+				ContentMD5: hash[:],
+			},
+		})
 	c.Assert(err, IsNil)
 }
 
@@ -333,7 +330,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
 
 	// 2nd link from pool, providing wrong path for source file
 	//
-	// this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
+	// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
 	s.prefixedStorage.pathCache = nil
 	err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
 	c.Check(err, IsNil)

cmd/cmd_test.go

@@ -1,88 +0,0 @@
-package cmd
-
-import (
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/deb"
-	"github.com/aptly-dev/aptly/utils"
-	"github.com/smira/flag"
-	. "gopkg.in/check.v1"
-)
-
-type CmdSuite struct {
-	mockProgress      *MockCmdProgress
-	collectionFactory *deb.CollectionFactory
-	mockContext       *MockCmdContext
-}
-
-var _ = Suite(&CmdSuite{})
-
-func (s *CmdSuite) SetUpTest(c *C) {
-	s.mockProgress = &MockCmdProgress{}
-
-	// Set up mock collections - use real collection factory
-	s.collectionFactory = deb.NewCollectionFactory(nil)
-
-	// Set up mock context
-	s.mockContext = &MockCmdContext{
-		progress:          s.mockProgress,
-		collectionFactory: s.collectionFactory,
-	}
-
-	// Skip setting mock context globally for type compatibility
-	// context = s.mockContext
-}
-
-func (s *CmdSuite) TestListPackagesRefListBasic(c *C) {
-	// Test basic functionality of ListPackagesRefList
-	// Need to initialize context for this test
-	if context == nil {
-		flags := flag.NewFlagSet("test", flag.ContinueOnError)
-		err := InitContext(flags)
-		c.Assert(err, IsNil)
-		defer ShutdownContext()
-	}
-	
-	reflist := &deb.PackageRefList{}
-
-	err := ListPackagesRefList(reflist, s.collectionFactory)
-	c.Check(err, IsNil)
-}
-
-func (s *CmdSuite) TestPrintPackageListBasic(c *C) {
-	// Test basic PrintPackageList functionality
-	packageList := deb.NewPackageList()
-
-	err := PrintPackageList(packageList, "", "  ")
-	c.Check(err, IsNil)
-}
-
-// Mock implementations for testing
-
-type MockCmdProgress struct {
-	messages []string
-}
-
-func (m *MockCmdProgress) Printf(msg string, a ...interface{})                      {}
-func (m *MockCmdProgress) ColoredPrintf(msg string, a ...interface{})               {}
-func (m *MockCmdProgress) PrintfStdErr(msg string, a ...interface{})                {}
-func (m *MockCmdProgress) Flush()                                                   {}
-func (m *MockCmdProgress) Start()                                                   {}
-func (m *MockCmdProgress) Shutdown()                                                {}
-func (m *MockCmdProgress) InitBar(count int64, isBytes bool, barType aptly.BarType) {}
-func (m *MockCmdProgress) ShutdownBar()                                             {}
-func (m *MockCmdProgress) AddBar(count int)                                         {}
-func (m *MockCmdProgress) SetBar(count int)                                         {}
-func (m *MockCmdProgress) PrintfBar(msg string, a ...interface{})                   {}
-func (m *MockCmdProgress) Write(p []byte) (n int, err error)                        { return len(p), nil }
-
-type MockCmdContext struct {
-	progress          *MockCmdProgress
-	collectionFactory *deb.CollectionFactory
-}
-
-func (m *MockCmdContext) Flags() *flag.FlagSet                         { return &flag.FlagSet{} }
-func (m *MockCmdContext) Progress() aptly.Progress                     { return m.progress }
-func (m *MockCmdContext) NewCollectionFactory() *deb.CollectionFactory { return m.collectionFactory }
-func (m *MockCmdContext) Config() *utils.ConfigStructure               { return &utils.ConfigStructure{} }
-
-// Note: Complex integration tests have been simplified for compilation compatibility.

cmd/context.go

@@ -9,16 +9,12 @@ var context *ctx.AptlyContext
 
 // ShutdownContext shuts context down
 func ShutdownContext() {
-	if context != nil {
-		context.Shutdown()
-	}
+	context.Shutdown()
 }
 
 // CleanupContext does partial shutdown of context
 func CleanupContext() {
-	if context != nil {
-		context.Cleanup()
-	}
+	context.Cleanup()
 }
 
 // InitContext initializes context with default settings

cmd/context_test.go

@@ -1,240 +0,0 @@
-package cmd
-
-import (
-	"testing"
-
-	ctx "github.com/aptly-dev/aptly/context"
-	"github.com/smira/flag"
-	. "gopkg.in/check.v1"
-)
-
-// Hook up gocheck into the "go test" runner.
-func Test(t *testing.T) { TestingT(t) }
-
-type ContextSuite struct {
-	originalContext *ctx.AptlyContext
-}
-
-var _ = Suite(&ContextSuite{})
-
-func (s *ContextSuite) SetUpTest(c *C) {
-	// Save original context state
-	s.originalContext = context
-	context = nil // Reset context for each test
-}
-
-func (s *ContextSuite) TearDownTest(c *C) {
-	// Clean up and restore original context
-	if context != nil {
-		context.Shutdown()
-		context = nil
-	}
-	context = s.originalContext
-}
-
-func (s *ContextSuite) TestInitContextSuccess(c *C) {
-	// Test successful context initialization
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-	c.Check(context, NotNil)
-	c.Check(GetContext(), Equals, context)
-}
-
-func (s *ContextSuite) TestInitContextPanic(c *C) {
-	// Test that initializing context twice causes panic
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	// First initialization should succeed
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-	c.Check(context, NotNil)
-
-	// Second initialization should panic
-	c.Check(func() { InitContext(flags) }, Panics, "context already initialized")
-}
-
-func (s *ContextSuite) TestInitContextError(c *C) {
-	// Test context initialization with invalid flags
-	// This tests the error path where ctx.NewContext might fail
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	// Add some invalid flag configuration that might cause NewContext to fail
-	// Note: This depends on the ctx.NewContext implementation details
-	flags.String("invalid-config", "/nonexistent/path/to/config", "invalid config")
-	flags.Set("invalid-config", "/nonexistent/path/to/config")
-
-	err := InitContext(flags)
-	// The error handling depends on the ctx.NewContext implementation
-	// If it doesn't fail with invalid paths, the test still validates the error path exists
-	if err != nil {
-		c.Check(context, IsNil)
-	} else {
-		c.Check(context, NotNil)
-	}
-}
-
-func (s *ContextSuite) TestGetContextBeforeInit(c *C) {
-	// Test GetContext when context is nil
-	c.Check(context, IsNil)
-	result := GetContext()
-	c.Check(result, IsNil)
-}
-
-func (s *ContextSuite) TestGetContextAfterInit(c *C) {
-	// Test GetContext after successful initialization
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-
-	result := GetContext()
-	c.Check(result, NotNil)
-	c.Check(result, Equals, context)
-}
-
-func (s *ContextSuite) TestShutdownContext(c *C) {
-	// Test ShutdownContext function
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-	c.Check(context, NotNil)
-
-	// ShutdownContext should not panic and should call context.Shutdown()
-	ShutdownContext() // Should not panic
-}
-
-func (s *ContextSuite) TestShutdownContextNil(c *C) {
-	// Test ShutdownContext when context is nil (should handle gracefully)
-	context = nil
-
-	// Should not panic when context is nil
-	ShutdownContext() // Should handle nil gracefully
-}
-
-func (s *ContextSuite) TestCleanupContext(c *C) {
-	// Test CleanupContext function
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-	c.Check(context, NotNil)
-
-	// CleanupContext should not panic and should call context.Cleanup()
-	CleanupContext() // Should not panic
-}
-
-func (s *ContextSuite) TestCleanupContextNil(c *C) {
-	// Test CleanupContext when context is nil (should handle gracefully)
-	context = nil
-
-	// Should not panic when context is nil
-	CleanupContext() // Should handle nil gracefully
-}
-
-func (s *ContextSuite) TestContextLifecycle(c *C) {
-	// Test complete context lifecycle: init -> use -> cleanup -> shutdown
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	// Initialize
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-	c.Check(context, NotNil)
-
-	// Use
-	ctx := GetContext()
-	c.Check(ctx, NotNil)
-	c.Check(ctx, Equals, context)
-
-	// Cleanup
-	CleanupContext() // Should not panic
-
-	// Context should still exist after cleanup
-	c.Check(context, NotNil)
-	c.Check(GetContext(), NotNil)
-
-	// Shutdown
-	ShutdownContext() // Should not panic
-}
-
-func (s *ContextSuite) TestMultipleCleanups(c *C) {
-	// Test calling CleanupContext multiple times
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-
-	// Multiple cleanups should not cause issues
-	CleanupContext() // First cleanup
-	CleanupContext() // Second cleanup
-	CleanupContext() // Third cleanup
-}
-
-func (s *ContextSuite) TestContextVariableIsolation(c *C) {
-	// Test that the context variable is properly managed
-	c.Check(context, IsNil)
-
-	flags := flag.NewFlagSet("test", flag.ContinueOnError)
-	err := InitContext(flags)
-	c.Check(err, IsNil)
-
-	// Store reference
-	originalContext := context
-	c.Check(originalContext, NotNil)
-
-	// GetContext should return the same instance
-	retrievedContext := GetContext()
-	c.Check(retrievedContext, Equals, originalContext)
-
-	// Context variable should be the same
-	c.Check(context, Equals, originalContext)
-}
-
-func (s *ContextSuite) TestFlagSetVariations(c *C) {
-	// Test InitContext with different FlagSet configurations
-	testCases := []struct {
-		name    string
-		setupFn func() *flag.FlagSet
-	}{
-		{
-			name: "empty flagset",
-			setupFn: func() *flag.FlagSet {
-				return flag.NewFlagSet("empty", flag.ContinueOnError)
-			},
-		},
-		{
-			name: "flagset with common flags",
-			setupFn: func() *flag.FlagSet {
-				fs := flag.NewFlagSet("common", flag.ContinueOnError)
-				fs.String("config", "", "config file")
-				fs.Bool("debug", false, "debug mode")
-				return fs
-			},
-		},
-		{
-			name: "flagset with aptly-specific flags",
-			setupFn: func() *flag.FlagSet {
-				fs := flag.NewFlagSet("aptly", flag.ContinueOnError)
-				fs.String("architectures", "", "architectures")
-				fs.String("distribution", "", "distribution")
-				return fs
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		// Reset context for each test case
-		if context != nil {
-			context.Shutdown()
-			context = nil
-		}
-
-		flags := tc.setupFn()
-		err := InitContext(flags)
-		c.Check(err, IsNil, Commentf("Failed for test case: %s", tc.name))
-		c.Check(context, NotNil, Commentf("Context is nil for test case: %s", tc.name))
-		c.Check(GetContext(), NotNil, Commentf("GetContext returned nil for test case: %s", tc.name))
-	}
-}

cmd/publish.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"strings"
+
 	"github.com/aptly-dev/aptly/pgp"
 	"github.com/smira/commander"
 	"github.com/smira/flag"
@@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 	}
 
 	signer := context.GetSigner()
-	signer.SetKey(flags.Lookup("gpg-key").Value.String())
+
+	var gpgKeys []string
+
+	// CLI args have priority over config
+	cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
+	if len(cliKeys) > 0 {
+		gpgKeys = cliKeys
+	} else if len(context.Config().GpgKeys) > 0 {
+		gpgKeys = context.Config().GpgKeys
+	}
+
+	for _, gpgKey := range gpgKeys {
+		signer.SetKey(gpgKey)
+	}
 	signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
 	signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
 	signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
 
 }
 
+type gpgKeyFlag struct {
+	gpgKeys []string
+}
+
+func (k *gpgKeyFlag) Set(value string) error {
+	k.gpgKeys = append(k.gpgKeys, value)
+	return nil
+}
+
+func (k *gpgKeyFlag) Get() interface{} {
+	return k.gpgKeys
+}
+
+func (k *gpgKeyFlag) String() string {
+	return strings.Join(k.gpgKeys, ",")
+}
+
 func makeCmdPublish() *commander.Command {
 	return &commander.Command{
 		UsageLine: "publish",

cmd/publish_repo.go

@@ -34,7 +34,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_snapshot.go

@@ -190,9 +190,11 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
 
 	context.Progress().Printf("\n%s been successfully published.\n", message)
 
-	if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok {
-		context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
-			localStorage.PublicPath())
+	if ps, err := context.GetPublishedStorage(storage); err == nil {
+		if localStorage, ok := ps.(aptly.FileSystemPublishedStorage); ok {
+			context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n",
+				localStorage.PublicPath())
+		}
 	}
 
 	context.Progress().Printf("Now you can add following line to apt sources:\n")
@@ -230,7 +232,7 @@ Example:
 	}
 	cmd.Flag.String("distribution", "", "distribution name to publish")
 	cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_switch.go

@@ -151,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/publish_update.go

@@ -115,7 +115,7 @@ Example:
 `,
 		Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
 	}
-	cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
+	cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
 	cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
 	cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
 	cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")

cmd/run.go

@@ -8,8 +8,8 @@ import (
 	"github.com/smira/commander"
 )
 
-// RunCommand runs single command starting from root cmd with args, optionally initializing context
-func RunCommand(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
+// Run runs single command starting from root cmd with args, optionally initializing context
+func Run(cmd *commander.Command, cmdArgs []string, initContext bool) (returnCode int) {
 	defer func() {
 		if r := recover(); r != nil {
 			fatal, ok := r.(*ctx.FatalError)

cmd/serve.go

@@ -97,7 +97,11 @@ func aptlyServe(cmd *commander.Command, args []string) error {
 		}
 	}
 
-	publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath()
+	ps, err := context.GetPublishedStorage("")
+	if err != nil {
+		return err
+	}
+	publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath()
 	ShutdownContext()
 
 	fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)

cmd/task_run.go

@@ -89,7 +89,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
 			context.Progress().ColoredPrintf("\n@yBegin command output: ----------------------------@!")
 			context.Progress().Flush()
 
-			returnCode := RunCommand(RootCommand(), command, false)
+			returnCode := Run(RootCommand(), command, false)
 			if returnCode != 0 {
 				commandErrored = true
 			}

console/progress.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"time"
 
 	"github.com/aptly-dev/aptly/aptly"
 	"github.com/aptly-dev/aptly/utils"
@@ -64,23 +63,8 @@ func (p *Progress) Start() {
 // Shutdown shuts down progress display
 func (p *Progress) Shutdown() {
 	p.ShutdownBar()
-	
-	// Send stop signal with timeout to prevent hanging
-	select {
-	case p.queue <- printTask{code: codeStop}:
-		// Successfully sent stop signal
-	case <-time.After(1 * time.Second):
-		// Timeout - queue might be full or nil
-		return
-	}
-	
-	// Wait for worker to stop with timeout
-	select {
-	case <-p.stopped:
-		// Worker stopped successfully
-	case <-time.After(1 * time.Second):
-		// Timeout - worker might be stuck
-	}
+	p.queue <- printTask{code: codeStop}
+	<-p.stopped
 }
 
 // Flush waits for all queued messages to be displayed
@@ -217,15 +201,7 @@ func (w *standardProgressWorker) run() {
 	hasBar := false
 
 	for {
-		task, ok := <-w.progress.queue
-		if !ok {
-			// Channel closed, exit gracefully
-			select {
-			case w.progress.stopped <- true:
-			default:
-			}
-			return
-		}
+		task := <-w.progress.queue
 		switch task.code {
 		case codeBarEnabled:
 			hasBar = true
@@ -270,15 +246,7 @@ func (w *loggerProgressWorker) run() {
 	hasBar := false
 
 	for {
-		task, ok := <-w.progress.queue
-		if !ok {
-			// Channel closed, exit gracefully
-			select {
-			case w.progress.stopped <- true:
-			default:
-			}
-			return
-		}
+		task := <-w.progress.queue
 		switch task.code {
 		case codeBarEnabled:
 			hasBar = true

context/context.go

@@ -100,6 +100,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
 
 			for _, configLocation := range configLocations {
+				// FIXME: check if exists, check if readable
 				err = utils.LoadConfig(configLocation, &utils.Config)
 				if os.IsPermission(err) || os.IsNotExist(err) {
 					continue
@@ -115,7 +116,12 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
 
-				_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
+				defaultConfig := aptly.AptlyConf
+				if len(defaultConfig) == 0 {
+					defaultConfig = []byte("root_dir: \"\"")
+				}
+
+				_ = utils.SaveConfigRaw(homeLocation, defaultConfig)
 				err = utils.LoadConfig(homeLocation, &utils.Config)
 				if err != nil {
 					Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -308,36 +314,7 @@ func (context *AptlyContext) _database() (database.Storage, error) {
 			}
 			context.database, err = goleveldb.NewDB(dbPath)
 		case "etcd":
-			// Configure etcd from config values
-			etcddb.ConfigureFromDBConfig(
-				context.config().DatabaseBackend.Timeout,
-				context.config().DatabaseBackend.WriteRetries,
-			)
-			
-			// Create queue config from settings
-			queueConfig := &etcddb.QueueConfig{
-				Enabled:         context.config().DatabaseBackend.WriteQueue.Enabled,
-				WriteQueueSize:  context.config().DatabaseBackend.WriteQueue.QueueSize,
-				MaxWritesPerSec: context.config().DatabaseBackend.WriteQueue.MaxWritesPerSec,
-				BatchMaxSize:    context.config().DatabaseBackend.WriteQueue.BatchMaxSize,
-				BatchMaxWait:    time.Duration(context.config().DatabaseBackend.WriteQueue.BatchMaxWaitMs) * time.Millisecond,
-			}
-			
-			// Set defaults if not configured
-			if queueConfig.WriteQueueSize == 0 {
-				queueConfig.WriteQueueSize = 1000
-			}
-			if queueConfig.MaxWritesPerSec == 0 {
-				queueConfig.MaxWritesPerSec = 100
-			}
-			if queueConfig.BatchMaxSize == 0 {
-				queueConfig.BatchMaxSize = 50
-			}
-			if queueConfig.BatchMaxWait == 0 {
-				queueConfig.BatchMaxWait = 10 * time.Millisecond
-			}
-			
-			context.database, err = etcddb.NewDBWithQueue(context.config().DatabaseBackend.URL, queueConfig)
+			context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
 		default:
 			context.database, err = goleveldb.NewDB(context.dbPath())
 		}
@@ -435,46 +412,26 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
 	return context.packagePool
 }
 
-// GetPublishedStorage returns instance of PublishedStorage
-func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage {
-	// Fast path: check if already exists without lock
-	context.Lock()
-	publishedStorage, ok := context.publishedStorages[name]
-	context.Unlock()
-
-	if ok {
-		return publishedStorage
-	}
-
-	// Slow path: need to create storage
+// GetPublishedStorage returns instance of PublishedStorage, or an error if the storage is not configured
+func (context *AptlyContext) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	context.Lock()
 	defer context.Unlock()
 
-	// Double-check after acquiring lock
-	publishedStorage, ok = context.publishedStorages[name]
-	if ok {
-		return publishedStorage
-	}
-
-	// Now safe to create new storage
-	if true { // Keep original indentation
+	publishedStorage, ok := context.publishedStorages[name]
+	if !ok {
 		if name == "" {
 			publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().GetRootDir(), "public"), "hardlink", "")
 		} else if strings.HasPrefix(name, "filesystem:") {
-			// Get a safe copy of the map
-			fileSystemRoots := context.config().GetFileSystemPublishRoots()
-			params, ok := fileSystemRoots[name[11:]]
+			params, ok := context.config().FileSystemPublishRoots[name[11:]]
 			if !ok {
-				Fatal(fmt.Errorf("published local storage %v not configured", name[11:]))
+				return nil, fmt.Errorf("published local storage %v not configured", name[11:])
 			}
 
 			publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod)
 		} else if strings.HasPrefix(name, "s3:") {
-			// Get a safe copy of the map
-			s3Roots := context.config().GetS3PublishRoots()
-			params, ok := s3Roots[name[3:]]
+			params, ok := context.config().S3PublishRoots[name[3:]]
 			if !ok {
-				Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:]))
+				return nil, fmt.Errorf("published S3 storage %v not configured", name[3:])
 			}
 
 			var err error
@@ -482,46 +439,41 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
 				params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
 				params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
 				params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
-				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug, params.ConcurrentUploads,
-				params.UploadQueueSize)
+				params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "swift:") {
-			// Get a safe copy of the map
-			swiftRoots := context.config().GetSwiftPublishRoots()
-			params, ok := swiftRoots[name[6:]]
+			params, ok := context.config().SwiftPublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Swift storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password,
 				params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else if strings.HasPrefix(name, "azure:") {
-			// Get a safe copy of the map
-			azureRoots := context.config().GetAzurePublishRoots()
-			params, ok := azureRoots[name[6:]]
+			params, ok := context.config().AzurePublishRoots[name[6:]]
 			if !ok {
-				Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:]))
+				return nil, fmt.Errorf("published Azure storage %v not configured", name[6:])
 			}
 
 			var err error
 			publishedStorage, err = azure.NewPublishedStorage(
 				params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint)
 			if err != nil {
-				Fatal(err)
+				return nil, err
 			}
 		} else {
-			Fatal(fmt.Errorf("unknown published storage format: %v", name))
+			return nil, fmt.Errorf("unknown published storage format: %v", name)
 		}
 		context.publishedStorages[name] = publishedStorage
 	}
 
-	return publishedStorage
+	return publishedStorage, nil
 }
 
 // UploadPath builds path to upload storage

context/context_race_test.go

@@ -1,179 +0,0 @@
-package context
-
-import (
-	"fmt"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/utils"
-)
-
-// Test for unsafe map access race condition
-func TestPublishedStorageMapRace(t *testing.T) {
-	// Create a context with empty config
-	context := &AptlyContext{
-		publishedStorages: make(map[string]aptly.PublishedStorage),
-		configLoaded:      true, // Skip config loading
-	}
-
-	// Mock config
-	utils.Config = utils.ConfigStructure{
-		RootDir: "/tmp/aptly-test",
-		FileSystemPublishRoots: map[string]utils.FileSystemPublishRoot{
-			"test": {RootDir: "/tmp/test", LinkMethod: "hardlink"},
-		},
-	}
-
-	var wg sync.WaitGroup
-	errors := make(chan error, 100)
-
-	// Simulate concurrent access to the same storage
-	for i := 0; i < 50; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					errors <- fmt.Errorf("panic in goroutine %d: %v", id, r)
-				}
-			}()
-
-			// All goroutines try to access the same storage
-			storage := context.GetPublishedStorage("filesystem:test")
-			if storage == nil {
-				errors <- fmt.Errorf("got nil storage in goroutine %d", id)
-			}
-		}(i)
-	}
-
-	// Also test different storages to trigger map growth
-	for i := 0; i < 10; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					errors <- fmt.Errorf("panic in storage %d: %v", id, r)
-				}
-			}()
-
-			// Add new storage configurations
-			storageName := fmt.Sprintf("filesystem:test%d", id)
-			utils.Config.SetFileSystemPublishRoot(fmt.Sprintf("test%d", id), utils.FileSystemPublishRoot{
-				RootDir:    fmt.Sprintf("/tmp/test%d", id),
-				LinkMethod: "hardlink",
-			})
-
-			storage := context.GetPublishedStorage(storageName)
-			if storage == nil {
-				errors <- fmt.Errorf("got nil storage for %s", storageName)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	close(errors)
-
-	// Check for any errors or panics
-	for err := range errors {
-		t.Errorf("Race condition error: %v", err)
-	}
-}
-
-// Test for concurrent map writes
-func TestPublishedStorageConcurrentWrites(t *testing.T) {
-	context := &AptlyContext{
-		publishedStorages: make(map[string]aptly.PublishedStorage),
-		configLoaded:      true, // Skip config loading
-	}
-
-	utils.Config = utils.ConfigStructure{
-		RootDir:                "/tmp/aptly-test",
-		FileSystemPublishRoots: make(map[string]utils.FileSystemPublishRoot),
-	}
-
-	var wg sync.WaitGroup
-	panics := make(chan string, 100)
-
-	// Multiple goroutines trying to create different storages simultaneously
-	for i := 0; i < 20; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panics <- fmt.Sprintf("goroutine %d panicked: %v", id, r)
-				}
-			}()
-
-			storageName := fmt.Sprintf("filesystem:concurrent%d", id)
-			utils.Config.SetFileSystemPublishRoot(fmt.Sprintf("concurrent%d", id), utils.FileSystemPublishRoot{
-				RootDir:    fmt.Sprintf("/tmp/concurrent%d", id),
-				LinkMethod: "hardlink",
-			})
-
-			// This should trigger concurrent map writes
-			_ = context.GetPublishedStorage(storageName)
-
-			// Add some delay to increase chance of race
-			time.Sleep(time.Millisecond)
-
-			// Access again to ensure consistency
-			storage2 := context.GetPublishedStorage(storageName)
-			if storage2 == nil {
-				panics <- fmt.Sprintf("inconsistent storage access in goroutine %d", id)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	close(panics)
-
-	// Check for panics (indicating race condition)
-	for panic := range panics {
-		t.Errorf("Concurrent map access issue: %s", panic)
-	}
-}
-
-// Test for storage initialization race
-func TestPublishedStorageInitRace(t *testing.T) {
-	// Run this test multiple times to increase chance of catching race
-	for attempt := 0; attempt < 10; attempt++ {
-		context := &AptlyContext{
-			publishedStorages: make(map[string]aptly.PublishedStorage),
-			configLoaded:      true, // Skip config loading
-		}
-
-		utils.Config = utils.ConfigStructure{
-			RootDir: "/tmp/aptly-test",
-			FileSystemPublishRoots: map[string]utils.FileSystemPublishRoot{
-				"race": {RootDir: "/tmp/race", LinkMethod: "hardlink"},
-			},
-		}
-
-		var wg sync.WaitGroup
-		storages := make([]aptly.PublishedStorage, 10)
-
-		// Multiple goroutines accessing the same non-existent storage
-		for i := 0; i < 10; i++ {
-			wg.Add(1)
-			go func(idx int) {
-				defer wg.Done()
-				storages[idx] = context.GetPublishedStorage("filesystem:race")
-			}(i)
-		}
-
-		wg.Wait()
-
-		// All should get the same storage instance
-		firstStorage := storages[0]
-		for i := 1; i < len(storages); i++ {
-			if storages[i] != firstStorage {
-				t.Errorf("Attempt %d: Got different storage instances: race condition in initialization", attempt)
-				break
-			}
-		}
-	}
-}

context/context_test.go

@@ -1,8 +1,6 @@
 package context
 
 import (
-	"fmt"
-	"os"
 	"reflect"
 	"testing"
 
@@ -80,10 +78,9 @@ func (s *AptlyContextSuite) SetUpTest(c *C) {
 
 func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) {
 	// https://github.com/aptly-dev/aptly/issues/711
-	// This will fail on account of us not having a config, so the
-	// storage never exists.
-	c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") },
-		FatalErrorPanicMatches,
-		&FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)",
-			os.Getenv("HOME"))})
+	// https://github.com/aptly-dev/aptly/issues/1477
+	// GetPublishedStorage must return an error (not panic) when the
+	// requested storage is not configured.
+	_, err := s.context.GetPublishedStorage("filesystem:fuji")
+	c.Assert(err, NotNil)
 }

context/queue_test.go

@@ -1,46 +0,0 @@
-package context
-
-import (
-	"testing"
-	"github.com/aptly-dev/aptly/utils"
-)
-
-func TestQueueConfigurationParsing(t *testing.T) {
-	// Test default configuration
-	config := utils.ConfigStructure{
-		DatabaseBackend: utils.DBConfig{
-			Type: "etcd",
-			URL:  "localhost:2379",
-		},
-	}
-
-	// Verify defaults are applied
-	if config.DatabaseBackend.WriteQueue.Enabled {
-		t.Error("Expected write queue to be disabled by default")
-	}
-
-	// Test with explicit configuration
-	config.DatabaseBackend.WriteQueue = utils.WriteQConfig{
-		Enabled:         true,
-		QueueSize:       500,
-		MaxWritesPerSec: 50,
-		BatchMaxSize:    25,
-		BatchMaxWaitMs:  20,
-	}
-
-	if !config.DatabaseBackend.WriteQueue.Enabled {
-		t.Error("Expected write queue to be enabled")
-	}
-	if config.DatabaseBackend.WriteQueue.QueueSize != 500 {
-		t.Errorf("Expected queue size 500, got %d", config.DatabaseBackend.WriteQueue.QueueSize)
-	}
-	if config.DatabaseBackend.WriteQueue.MaxWritesPerSec != 50 {
-		t.Errorf("Expected max writes per sec 50, got %d", config.DatabaseBackend.WriteQueue.MaxWritesPerSec)
-	}
-	if config.DatabaseBackend.WriteQueue.BatchMaxSize != 25 {
-		t.Errorf("Expected batch max size 25, got %d", config.DatabaseBackend.WriteQueue.BatchMaxSize)
-	}
-	if config.DatabaseBackend.WriteQueue.BatchMaxWaitMs != 20 {
-		t.Errorf("Expected batch max wait 20ms, got %d", config.DatabaseBackend.WriteQueue.BatchMaxWaitMs)
-	}
-}

database/database_test.go

@@ -1,210 +0,0 @@
-package database
-
-import (
-	"errors"
-	"testing"
-
-	check "gopkg.in/check.v1"
-)
-
-// Hook up gocheck into the "go test" runner.
-func Test(t *testing.T) { check.TestingT(t) }
-
-type DatabaseSuite struct{}
-
-var _ = check.Suite(&DatabaseSuite{})
-
-func (s *DatabaseSuite) TestErrNotFound(c *check.C) {
-	// Test that ErrNotFound is properly defined
-	c.Check(ErrNotFound, check.NotNil)
-	c.Check(ErrNotFound.Error(), check.Equals, "key not found")
-
-	// Test that it's an actual error
-	var err error = ErrNotFound
-	c.Check(err, check.NotNil)
-
-	// Test comparison with errors.New
-	newErr := errors.New("key not found")
-	c.Check(ErrNotFound.Error(), check.Equals, newErr.Error())
-
-	// Test that it's not equal to other errors
-	otherErr := errors.New("other error")
-	c.Check(ErrNotFound.Error(), check.Not(check.Equals), otherErr.Error())
-}
-
-func (s *DatabaseSuite) TestStorageProcessor(c *check.C) {
-	// Test StorageProcessor function type
-	called := false
-	var processor StorageProcessor = func(key []byte, value []byte) error {
-		called = true
-		c.Check(key, check.DeepEquals, []byte("test-key"))
-		c.Check(value, check.DeepEquals, []byte("test-value"))
-		return nil
-	}
-
-	err := processor([]byte("test-key"), []byte("test-value"))
-	c.Check(err, check.IsNil)
-	c.Check(called, check.Equals, true)
-}
-
-func (s *DatabaseSuite) TestStorageProcessorWithError(c *check.C) {
-	// Test StorageProcessor that returns an error
-	testError := errors.New("processing error")
-	var processor StorageProcessor = func(key []byte, value []byte) error {
-		return testError
-	}
-
-	err := processor([]byte("key"), []byte("value"))
-	c.Check(err, check.Equals, testError)
-}
-
-func (s *DatabaseSuite) TestStorageProcessorNilInputs(c *check.C) {
-	// Test StorageProcessor with nil inputs
-	var processor StorageProcessor = func(key []byte, value []byte) error {
-		c.Check(key, check.IsNil)
-		c.Check(value, check.DeepEquals, []byte("value"))
-		return nil
-	}
-
-	err := processor(nil, []byte("value"))
-	c.Check(err, check.IsNil)
-}
-
-func (s *DatabaseSuite) TestStorageProcessorEmptyInputs(c *check.C) {
-	// Test StorageProcessor with empty inputs
-	var processor StorageProcessor = func(key []byte, value []byte) error {
-		c.Check(len(key), check.Equals, 0)
-		c.Check(len(value), check.Equals, 0)
-		return nil
-	}
-
-	err := processor([]byte{}, []byte{})
-	c.Check(err, check.IsNil)
-}
-
-// Mock implementations to test interface compliance
-type mockReader struct {
-	data map[string][]byte
-}
-
-func (m *mockReader) Get(key []byte) ([]byte, error) {
-	if value, exists := m.data[string(key)]; exists {
-		return value, nil
-	}
-	return nil, ErrNotFound
-}
-
-type mockWriter struct {
-	data map[string][]byte
-}
-
-func (m *mockWriter) Put(key []byte, value []byte) error {
-	m.data[string(key)] = value
-	return nil
-}
-
-func (m *mockWriter) Delete(key []byte) error {
-	delete(m.data, string(key))
-	return nil
-}
-
-type mockReaderWriter struct {
-	*mockReader
-	*mockWriter
-}
-
-func (s *DatabaseSuite) TestReaderInterface(c *check.C) {
-	// Test Reader interface implementation
-	data := map[string][]byte{
-		"key1": []byte("value1"),
-		"key2": []byte("value2"),
-	}
-
-	var reader Reader = &mockReader{data: data}
-
-	// Test existing key
-	value, err := reader.Get([]byte("key1"))
-	c.Check(err, check.IsNil)
-	c.Check(value, check.DeepEquals, []byte("value1"))
-
-	// Test non-existing key
-	value, err = reader.Get([]byte("nonexistent"))
-	c.Check(err, check.Equals, ErrNotFound)
-	c.Check(value, check.IsNil)
-}
-
-func (s *DatabaseSuite) TestWriterInterface(c *check.C) {
-	// Test Writer interface implementation
-	data := make(map[string][]byte)
-	var writer Writer = &mockWriter{data: data}
-
-	// Test Put
-	err := writer.Put([]byte("key1"), []byte("value1"))
-	c.Check(err, check.IsNil)
-	c.Check(data["key1"], check.DeepEquals, []byte("value1"))
-
-	// Test Delete
-	err = writer.Delete([]byte("key1"))
-	c.Check(err, check.IsNil)
-	_, exists := data["key1"]
-	c.Check(exists, check.Equals, false)
-}
-
-func (s *DatabaseSuite) TestReaderWriterInterface(c *check.C) {
-	// Test ReaderWriter interface implementation
-	data := make(map[string][]byte)
-
-	var rw ReaderWriter = &mockReaderWriter{
-		mockReader: &mockReader{data: data},
-		mockWriter: &mockWriter{data: data},
-	}
-
-	// Test write then read
-	err := rw.Put([]byte("test"), []byte("value"))
-	c.Check(err, check.IsNil)
-
-	value, err := rw.Get([]byte("test"))
-	c.Check(err, check.IsNil)
-	c.Check(value, check.DeepEquals, []byte("value"))
-
-	// Test delete
-	err = rw.Delete([]byte("test"))
-	c.Check(err, check.IsNil)
-
-	value, err = rw.Get([]byte("test"))
-	c.Check(err, check.Equals, ErrNotFound)
-	c.Check(value, check.IsNil)
-}
-
-// Test that all interfaces are properly defined
-func (s *DatabaseSuite) TestInterfaceDefinitions(c *check.C) {
-	// This test ensures that all interfaces are properly defined
-	// and can be used as interface types
-
-	var reader Reader
-	var prefixReader PrefixReader
-	var writer Writer
-	var readerWriter ReaderWriter
-	var storage Storage
-	var batch Batch
-	var transaction Transaction
-
-	// Test that they are nil by default
-	c.Check(reader, check.IsNil)
-	c.Check(prefixReader, check.IsNil)
-	c.Check(writer, check.IsNil)
-	c.Check(readerWriter, check.IsNil)
-	c.Check(storage, check.IsNil)
-	c.Check(batch, check.IsNil)
-	c.Check(transaction, check.IsNil)
-}
-
-func (s *DatabaseSuite) TestErrorConstants(c *check.C) {
-	// Test that error constants are immutable and consistently defined
-	original := ErrNotFound
-	c.Check(original, check.NotNil)
-
-	// Verify it maintains its identity
-	c.Check(ErrNotFound, check.Equals, original)
-	c.Check(ErrNotFound.Error(), check.Equals, original.Error())
-}

database/etcddb/batch.go

@@ -1,16 +1,8 @@
 package etcddb
 
 import (
-	"context"
-	"fmt"
-	"math"
-	"time"
-
 	"github.com/aptly-dev/aptly/database"
-	"github.com/rs/zerolog/log"
 	clientv3 "go.etcd.io/etcd/client/v3"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 )
 
 type EtcDBatch struct {
@@ -34,121 +26,25 @@ func (b *EtcDBatch) Delete(key []byte) (err error) {
 }
 
 func (b *EtcDBatch) Write() (err error) {
-	var kv clientv3.KV
-	if b.s.queuedKV != nil {
-		kv = b.s.queuedKV
-	} else {
-		kv = clientv3.NewKV(b.s.db)
-	}
+	kv := clientv3.NewKV(b.s.db)
 
 	batchSize := 128
 	for i := 0; i < len(b.ops); i += batchSize {
+		txn := kv.Txn(Ctx)
 		end := i + batchSize
 		if end > len(b.ops) {
 			end = len(b.ops)
 		}
 
 		batch := b.ops[i:end]
-
-		// Retry logic with exponential backoff
-		var lastErr error
-		for retry := 0; retry <= DefaultWriteRetries; retry++ {
-			ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout)
-			txn := kv.Txn(ctx)
-			txn.Then(batch...)
-			_, err = txn.Commit()
-			cancel()
-
-			if err == nil {
-				// Success, move to next batch
-				break
-			}
-
-			lastErr = err
-
-			// Check if error is retryable
-			if !isRetryableError(err) {
-				log.Error().Err(err).Int("batch_start", i).Int("batch_end", end).Msg("etcd: non-retryable error during batch write")
-				return fmt.Errorf("etcd batch write failed: %w", err)
-			}
-
-			if retry < DefaultWriteRetries {
-				// Calculate exponential backoff
-				backoff := time.Duration(math.Pow(2, float64(retry))) * 100 * time.Millisecond
-				if backoff > 5*time.Second {
-					backoff = 5 * time.Second
-				}
-
-				log.Warn().Err(err).
-					Int("retry", retry+1).
-					Int("max_retries", DefaultWriteRetries).
-					Dur("backoff", backoff).
-					Int("batch_start", i).
-					Int("batch_end", end).
-					Msg("etcd: batch write failed, retrying")
-
-				time.Sleep(backoff)
-			}
-		}
-
-		// All retries exhausted
-		if lastErr != nil {
-			log.Error().Err(lastErr).
-				Int("batch_start", i).
-				Int("batch_end", end).
-				Int("retries", DefaultWriteRetries).
-				Msg("etcd: batch write failed after all retries")
-			return fmt.Errorf("etcd batch write failed after %d retries: %w", DefaultWriteRetries, lastErr)
+		txn.Then(batch...)
+		_, err = txn.Commit()
+		if err != nil {
+			panic(err)
 		}
 	}
 
-	return nil
-}
-
-// isRetryableError checks if an error is retryable
-func isRetryableError(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	// Check for gRPC status errors
-	if s, ok := status.FromError(err); ok {
-		switch s.Code() {
-		case codes.Unavailable, codes.DeadlineExceeded, codes.ResourceExhausted, codes.Aborted:
-			return true
-		}
-	}
-
-	// Check for context errors
-	if err == context.DeadlineExceeded || err == context.Canceled {
-		return true
-	}
-
-	// Check for timeout errors in error message
-	if errStr := err.Error(); errStr != "" {
-		if contains(errStr, "timeout") || contains(errStr, "timed out") ||
-			contains(errStr, "unavailable") || contains(errStr, "connection refused") {
-			return true
-		}
-	}
-
-	return false
-}
-
-// contains is a simple string contains helper
-func contains(s, substr string) bool {
-	return len(substr) > 0 && len(s) >= len(substr) &&
-		(s == substr || s[:len(substr)] == substr || s[len(s)-len(substr):] == substr ||
-			len(s) > len(substr) && findSubstring(s, substr))
-}
-
-func findSubstring(s, substr string) bool {
-	for i := 0; i <= len(s)-len(substr); i++ {
-		if s[i:i+len(substr)] == substr {
-			return true
-		}
-	}
-	return false
+	return
 }
 
 // batch should implement database.Batch

database/etcddb/database.go

@@ -1,83 +1,24 @@
 package etcddb
 
 import (
-	"os"
-	"strconv"
+	"context"
 	"time"
 
 	"github.com/aptly-dev/aptly/database"
-	"github.com/rs/zerolog/log"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
-// Default timeout for etcd operations
-var DefaultTimeout = 60 * time.Second
-
-// Default write retry count
-var DefaultWriteRetries = 3
-
-func init() {
-	// Allow timeout configuration via environment variable
-	if timeout := os.Getenv("APTLY_ETCD_TIMEOUT"); timeout != "" {
-		if d, err := time.ParseDuration(timeout); err == nil {
-			DefaultTimeout = d
-			log.Info().Dur("timeout", d).Msg("etcd: using custom timeout")
-		} else {
-			log.Warn().Str("value", timeout).Err(err).Msg("etcd: invalid timeout value, using default")
-		}
-	}
-
-	// Allow write retry configuration via environment variable
-	if retries := os.Getenv("APTLY_ETCD_WRITE_RETRIES"); retries != "" {
-		if r, err := strconv.Atoi(retries); err == nil && r >= 0 {
-			DefaultWriteRetries = r
-			log.Info().Int("retries", r).Msg("etcd: using custom write retry count")
-		} else {
-			log.Warn().Str("value", retries).Err(err).Msg("etcd: invalid write retry value, using default")
-		}
-	}
-}
+var Ctx = context.TODO()
 
 func internalOpen(url string) (cli *clientv3.Client, err error) {
-	// Configure dial timeout
-	dialTimeout := 60 * time.Second
-	if dt := os.Getenv("APTLY_ETCD_DIAL_TIMEOUT"); dt != "" {
-		if d, err := time.ParseDuration(dt); err == nil {
-			dialTimeout = d
-		}
-	}
-
-	// Configure keep alive timeout
-	keepAliveTimeout := 7200 * time.Second
-	if ka := os.Getenv("APTLY_ETCD_KEEPALIVE"); ka != "" {
-		if d, err := time.ParseDuration(ka); err == nil {
-			keepAliveTimeout = d
-		}
-	}
-
-	// Configure message size
-	maxMsgSize := 50 * 1024 * 1024 // 50MiB default
-	if size := os.Getenv("APTLY_ETCD_MAX_MSG_SIZE"); size != "" {
-		if s, err := strconv.Atoi(size); err == nil && s > 0 {
-			maxMsgSize = s
-		}
-	}
-
 	cfg := clientv3.Config{
 		Endpoints:            []string{url},
-		DialTimeout:          dialTimeout,
-		MaxCallSendMsgSize:   maxMsgSize,
-		MaxCallRecvMsgSize:   maxMsgSize,
-		DialKeepAliveTimeout: keepAliveTimeout,
+		DialTimeout:          30 * time.Second,
+		MaxCallSendMsgSize:   2147483647, // (2048 * 1024 * 1024) - 1
+		MaxCallRecvMsgSize:   2147483647,
+		DialKeepAliveTimeout: 7200 * time.Second,
 	}
 
-	log.Info().
-		Str("endpoint", url).
-		Dur("dialTimeout", dialTimeout).
-		Dur("keepAlive", keepAliveTimeout).
-		Int("maxMsgSize", maxMsgSize).
-		Msg("etcd: opening connection")
-
 	cli, err = clientv3.New(cfg)
 	return
 }
@@ -87,56 +28,5 @@ func NewDB(url string) (database.Storage, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &EtcDStorage{
-		url:          url,
-		db:           cli,
-		queuedClient: nil,
-		queuedKV:     nil,
-		tmpPrefix:    "",
-	}, nil
-}
-
-// NewDBWithQueue creates a new DB with optional write queue
-func NewDBWithQueue(url string, queueConfig *QueueConfig) (database.Storage, error) {
-	cli, err := internalOpen(url)
-	if err != nil {
-		return nil, err
-	}
-	
-	storage := &EtcDStorage{
-		url:       url,
-		db:        cli,
-		tmpPrefix: "",
-	}
-	
-	if queueConfig != nil && queueConfig.Enabled {
-		storage.queuedClient = NewQueuedEtcdClient(cli, queueConfig)
-		storage.queuedKV = NewQueuedKV(cli.KV, storage.queuedClient.writeQueue, queueConfig)
-		log.Info().
-			Bool("enabled", queueConfig.Enabled).
-			Int("queueSize", queueConfig.WriteQueueSize).
-			Int("maxWritesPerSec", queueConfig.MaxWritesPerSec).
-			Msg("etcd: write queue enabled")
-	}
-	
-	return storage, nil
-}
-
-// ConfigureFromDBConfig applies configuration from DBConfig
-func ConfigureFromDBConfig(timeout string, writeRetries int) {
-	// Configure timeout if provided
-	if timeout != "" {
-		if d, err := time.ParseDuration(timeout); err == nil {
-			DefaultTimeout = d
-			log.Info().Dur("timeout", d).Msg("etcd: configured timeout from config")
-		} else {
-			log.Warn().Str("value", timeout).Err(err).Msg("etcd: invalid timeout in config, keeping current value")
-		}
-	}
-
-	// Configure write retries if provided
-	if writeRetries > 0 {
-		DefaultWriteRetries = writeRetries
-		log.Info().Int("retries", writeRetries).Msg("etcd: configured write retries from config")
-	}
+	return &EtcDStorage{url, cli, ""}, nil
 }

database/etcddb/integration_test.go

@@ -1,99 +0,0 @@
-package etcddb
-
-import (
-	"testing"
-	"time"
-	"github.com/aptly-dev/aptly/database"
-)
-
-func TestEtcdWithQueue(t *testing.T) {
-	// Test with queue enabled
-	config := &QueueConfig{
-		Enabled:         true,
-		WriteQueueSize:  100,
-		MaxWritesPerSec: 100,
-		BatchMaxSize:    10,
-		BatchMaxWait:    10 * time.Millisecond,
-	}
-
-	db, err := NewDBWithQueue("localhost:2379", config)
-	if err != nil {
-		t.Skipf("etcd not available: %v", err)
-	}
-	defer db.Close()
-
-	// Test basic operations
-	testKey := []byte("test-queue-key")
-	testValue := []byte("test-queue-value")
-
-	err = db.Put(testKey, testValue)
-	if err != nil {
-		t.Fatalf("Put failed: %v", err)
-	}
-
-	// Give queue time to process
-	time.Sleep(100 * time.Millisecond)
-
-	retrieved, err := db.Get(testKey)
-	if err != nil {
-		t.Fatalf("Get failed: %v", err)
-	}
-
-	if string(retrieved) != string(testValue) {
-		t.Fatalf("Expected %s, got %s", testValue, retrieved)
-	}
-
-	// Clean up
-	err = db.Delete(testKey)
-	if err != nil {
-		t.Fatalf("Delete failed: %v", err)
-	}
-}
-
-func TestEtcdWithoutQueue(t *testing.T) {
-	// Test with queue disabled
-	config := &QueueConfig{
-		Enabled: false,
-	}
-
-	db, err := NewDBWithQueue("localhost:2379", config)
-	if err != nil {
-		t.Skipf("etcd not available: %v", err)
-	}
-	defer db.Close()
-
-	// Verify it's regular etcd storage
-	_, ok := db.(*EtcDStorage)
-	if !ok {
-		t.Fatal("Expected EtcDStorage type")
-	}
-
-	// Test basic operations
-	testKey := []byte("test-no-queue-key")
-	testValue := []byte("test-no-queue-value")
-
-	err = db.Put(testKey, testValue)
-	if err != nil {
-		t.Fatalf("Put failed: %v", err)
-	}
-
-	retrieved, err := db.Get(testKey)
-	if err != nil {
-		t.Fatalf("Get failed: %v", err)
-	}
-
-	if string(retrieved) != string(testValue) {
-		t.Fatalf("Expected %s, got %s", testValue, retrieved)
-	}
-
-	// Clean up
-	err = db.Delete(testKey)
-	if err != nil {
-		t.Fatalf("Delete failed: %v", err)
-	}
-}
-
-func TestQueueImplementsInterface(t *testing.T) {
-	// Verify that our implementation satisfies the database.Storage interface
-	var _ database.Storage = (*EtcDStorage)(nil)
-}

database/etcddb/queue.go

@@ -1,381 +0,0 @@
-package etcddb
-
-import (
-	"context"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	clientv3 "go.etcd.io/etcd/client/v3"
-	"github.com/rs/zerolog/log"
-)
-
-// QueueConfig contains configuration for the write queue
-type QueueConfig struct {
-	Enabled         bool
-	WriteQueueSize  int
-	MaxWritesPerSec int
-	BatchMaxSize    int
-	BatchMaxWait    time.Duration
-}
-
-// DefaultQueueConfig returns default queue configuration
-func DefaultQueueConfig() *QueueConfig {
-	return &QueueConfig{
-		Enabled:         false,
-		WriteQueueSize:  1000,
-		MaxWritesPerSec: 100,
-		BatchMaxSize:    50,
-		BatchMaxWait:    10 * time.Millisecond,
-	}
-}
-
-// writeOp represents a queued write operation
-type writeOp struct {
-	fn     func() error
-	result chan error
-}
-
-// QueuedEtcdClient wraps an etcd client with write queueing
-type QueuedEtcdClient struct {
-	client     *clientv3.Client
-	kv         clientv3.KV
-	writeQueue chan writeOp
-	config     *QueueConfig
-	wg         sync.WaitGroup
-	done       chan struct{}
-	closed     atomic.Bool
-}
-
-// NewQueuedEtcdClient creates a new queued etcd client
-func NewQueuedEtcdClient(client *clientv3.Client, config *QueueConfig) *QueuedEtcdClient {
-	if config == nil {
-		config = DefaultQueueConfig()
-	}
-
-	qc := &QueuedEtcdClient{
-		client:     client,
-		kv:         client.KV,
-		writeQueue: make(chan writeOp, config.WriteQueueSize),
-		config:     config,
-		done:       make(chan struct{}),
-	}
-
-	if config.Enabled {
-		qc.wg.Add(1)
-		go qc.processQueue()
-	}
-
-	return qc
-}
-
-// processQueue processes write operations sequentially
-func (qc *QueuedEtcdClient) processQueue() {
-	defer qc.wg.Done()
-
-	ticker := time.NewTicker(time.Second / time.Duration(qc.config.MaxWritesPerSec))
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-qc.done:
-			// Cancel remaining operations
-			for len(qc.writeQueue) > 0 {
-				select {
-				case op := <-qc.writeQueue:
-					op.result <- context.Canceled
-				default:
-					return
-				}
-			}
-			return
-
-		case op := <-qc.writeQueue:
-			if qc.closed.Load() {
-				op.result <- context.Canceled
-				continue
-			}
-			qc.executeOp(op)
-			<-ticker.C // Rate limiting after operation
-		}
-	}
-}
-
-// executeOp executes a single write operation
-func (qc *QueuedEtcdClient) executeOp(op writeOp) {
-	start := time.Now()
-	err := op.fn()
-	duration := time.Since(start)
-
-	if err != nil {
-		log.Warn().Err(err).Dur("duration", duration).Msg("etcd write operation failed")
-	} else {
-		log.Debug().Dur("duration", duration).Msg("etcd write operation completed")
-	}
-
-	op.result <- err
-}
-
-// Close closes the queued client
-func (qc *QueuedEtcdClient) Close() error {
-	if qc.config.Enabled {
-		qc.closed.Store(true)
-		close(qc.done)
-		
-		// Wait for queue to drain with timeout
-		done := make(chan struct{})
-		go func() {
-			qc.wg.Wait()
-			close(done)
-		}()
-		
-		select {
-		case <-done:
-			// Queue drained successfully
-		case <-time.After(5 * time.Second):
-			// Timeout - log warning but continue
-			log.Warn().Msg("etcd: queue close timeout, some operations may be lost")
-		}
-	}
-	return qc.client.Close()
-}
-
-// QueuedKV implements clientv3.KV with write queueing
-type QueuedKV struct {
-	kv         clientv3.KV
-	writeQueue chan writeOp
-	config     *QueueConfig
-}
-
-// NewQueuedKV creates a new queued KV interface
-func NewQueuedKV(kv clientv3.KV, writeQueue chan writeOp, config *QueueConfig) *QueuedKV {
-	return &QueuedKV{
-		kv:         kv,
-		writeQueue: writeQueue,
-		config:     config,
-	}
-}
-
-// Put queues a put operation
-func (qkv *QueuedKV) Put(ctx context.Context, key, val string, opts ...clientv3.OpOption) (*clientv3.PutResponse, error) {
-	if !qkv.config.Enabled {
-		return qkv.kv.Put(ctx, key, val, opts...)
-	}
-
-	resultChan := make(chan error, 1)
-	respChan := make(chan *clientv3.PutResponse, 1)
-
-	select {
-	case qkv.writeQueue <- writeOp{
-		fn: func() error {
-			resp, err := qkv.kv.Put(ctx, key, val, opts...)
-			if err == nil {
-				respChan <- resp
-			}
-			return err
-		},
-		result: resultChan,
-	}:
-		// Successfully queued
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-
-	select {
-	case err := <-resultChan:
-		if err != nil {
-			return nil, err
-		}
-		return <-respChan, nil
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-}
-
-// Get performs a get operation (not queued)
-func (qkv *QueuedKV) Get(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
-	return qkv.kv.Get(ctx, key, opts...)
-}
-
-// Delete queues a delete operation
-func (qkv *QueuedKV) Delete(ctx context.Context, key string, opts ...clientv3.OpOption) (*clientv3.DeleteResponse, error) {
-	if !qkv.config.Enabled {
-		return qkv.kv.Delete(ctx, key, opts...)
-	}
-
-	resultChan := make(chan error, 1)
-	respChan := make(chan *clientv3.DeleteResponse, 1)
-
-	select {
-	case qkv.writeQueue <- writeOp{
-		fn: func() error {
-			resp, err := qkv.kv.Delete(ctx, key, opts...)
-			if err == nil {
-				respChan <- resp
-			}
-			return err
-		},
-		result: resultChan,
-	}:
-		// Successfully queued
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-
-	select {
-	case err := <-resultChan:
-		if err != nil {
-			return nil, err
-		}
-		return <-respChan, nil
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-}
-
-// Txn creates a transaction (will be queued)
-func (qkv *QueuedKV) Txn(ctx context.Context) clientv3.Txn {
-	return &QueuedTxn{
-		txn:        qkv.kv.Txn(ctx),
-		writeQueue: qkv.writeQueue,
-		config:     qkv.config,
-		ctx:        ctx,
-	}
-}
-
-// Do performs a generic operation
-func (qkv *QueuedKV) Do(ctx context.Context, op clientv3.Op) (clientv3.OpResponse, error) {
-	// Determine if this is a write operation
-	if op.IsGet() {
-		// Read operations are not queued
-		return qkv.kv.Do(ctx, op)
-	}
-
-	if !qkv.config.Enabled {
-		return qkv.kv.Do(ctx, op)
-	}
-
-	// Queue write operations
-	resultChan := make(chan error, 1)
-	respChan := make(chan clientv3.OpResponse, 1)
-
-	select {
-	case qkv.writeQueue <- writeOp{
-		fn: func() error {
-			resp, err := qkv.kv.Do(ctx, op)
-			if err == nil {
-				respChan <- resp
-			}
-			return err
-		},
-		result: resultChan,
-	}:
-		// Successfully queued
-	case <-ctx.Done():
-		return clientv3.OpResponse{}, ctx.Err()
-	}
-
-	err := <-resultChan
-	if err != nil {
-		return clientv3.OpResponse{}, err
-	}
-	return <-respChan, nil
-}
-
-// Compact queues a compact operation
-func (qkv *QueuedKV) Compact(ctx context.Context, rev int64, opts ...clientv3.CompactOption) (*clientv3.CompactResponse, error) {
-	if !qkv.config.Enabled {
-		return qkv.kv.Compact(ctx, rev, opts...)
-	}
-
-	resultChan := make(chan error, 1)
-	respChan := make(chan *clientv3.CompactResponse, 1)
-
-	select {
-	case qkv.writeQueue <- writeOp{
-		fn: func() error {
-			resp, err := qkv.kv.Compact(ctx, rev, opts...)
-			if err == nil {
-				respChan <- resp
-			}
-			return err
-		},
-		result: resultChan,
-	}:
-		// Successfully queued
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-
-	select {
-	case err := <-resultChan:
-		if err != nil {
-			return nil, err
-		}
-		return <-respChan, nil
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
-}
-
-// QueuedTxn wraps a transaction with queueing
-type QueuedTxn struct {
-	txn        clientv3.Txn
-	writeQueue chan writeOp
-	config     *QueueConfig
-	ctx        context.Context
-}
-
-// If sets the comparison target
-func (qtxn *QueuedTxn) If(cs ...clientv3.Cmp) clientv3.Txn {
-	qtxn.txn = qtxn.txn.If(cs...)
-	return qtxn
-}
-
-// Then sets the success operations
-func (qtxn *QueuedTxn) Then(ops ...clientv3.Op) clientv3.Txn {
-	qtxn.txn = qtxn.txn.Then(ops...)
-	return qtxn
-}
-
-// Else sets the failure operations
-func (qtxn *QueuedTxn) Else(ops ...clientv3.Op) clientv3.Txn {
-	qtxn.txn = qtxn.txn.Else(ops...)
-	return qtxn
-}
-
-// Commit queues the transaction commit
-func (qtxn *QueuedTxn) Commit() (*clientv3.TxnResponse, error) {
-	if !qtxn.config.Enabled {
-		return qtxn.txn.Commit()
-	}
-
-	resultChan := make(chan error, 1)
-	respChan := make(chan *clientv3.TxnResponse, 1)
-
-	select {
-	case qtxn.writeQueue <- writeOp{
-		fn: func() error {
-			resp, err := qtxn.txn.Commit()
-			if err == nil {
-				respChan <- resp
-			}
-			return err
-		},
-		result: resultChan,
-	}:
-		// Successfully queued
-	case <-qtxn.ctx.Done():
-		return nil, qtxn.ctx.Err()
-	}
-
-	select {
-	case err := <-resultChan:
-		if err != nil {
-			return nil, err
-		}
-		return <-respChan, nil
-	case <-qtxn.ctx.Done():
-		return nil, qtxn.ctx.Err()
-	}
-}

database/etcddb/queue_test.go

@@ -1,384 +0,0 @@
-package etcddb
-
-import (
-	"context"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	clientv3 "go.etcd.io/etcd/client/v3"
-	. "gopkg.in/check.v1"
-)
-
-type QueueSuite struct {
-	client *clientv3.Client
-	config *QueueConfig
-}
-
-var _ = Suite(&QueueSuite{})
-
-func TestQueue(t *testing.T) { TestingT(t) }
-
-func (s *QueueSuite) SetUpSuite(c *C) {
-	// Create a test etcd client
-	cli, err := clientv3.New(clientv3.Config{
-		Endpoints:   []string{"localhost:2379"},
-		DialTimeout: 5 * time.Second,
-	})
-	if err != nil {
-		c.Skip("etcd not available: " + err.Error())
-	}
-	s.client = cli
-}
-
-func (s *QueueSuite) TearDownSuite(c *C) {
-	if s.client != nil {
-		s.client.Close()
-	}
-}
-
-func (s *QueueSuite) SetUpTest(c *C) {
-	s.config = &QueueConfig{
-		Enabled:         true,
-		WriteQueueSize:  100,
-		MaxWritesPerSec: 100,  // Faster for tests
-		BatchMaxSize:    10,
-		BatchMaxWait:    10 * time.Millisecond,
-	}
-}
-
-func (s *QueueSuite) TearDownTest(c *C) {
-	// Clean up all test data
-	ctx := context.Background()
-	resp, err := s.client.Get(ctx, "/test/", clientv3.WithPrefix())
-	if err == nil && len(resp.Kvs) > 0 {
-		_, _ = s.client.Delete(ctx, "/test/", clientv3.WithPrefix())
-	}
-}
-
-func (s *QueueSuite) TestQueuedClientCreation(c *C) {
-	qc := NewQueuedEtcdClient(s.client, s.config)
-	c.Assert(qc, NotNil)
-	c.Assert(qc.client, Equals, s.client)
-	c.Assert(qc.config, DeepEquals, s.config)
-	c.Assert(cap(qc.writeQueue), Equals, 100)
-	
-	err := qc.Close()
-	c.Assert(err, IsNil)
-}
-
-func (s *QueueSuite) TestQueuedKVPut(c *C) {
-	qc := NewQueuedEtcdClient(s.client, s.config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, s.config)
-	
-	ctx := context.Background()
-	key := "/test/queue/put"
-	value := "test-value"
-	
-	// Clean up first
-	s.client.Delete(ctx, key)
-	
-	// Put via queued KV
-	_, err := qkv.Put(ctx, key, value)
-	c.Assert(err, IsNil)
-	
-	// Give queue time to process
-	time.Sleep(200 * time.Millisecond)
-	
-	// Verify via direct client
-	resp, err := s.client.Get(ctx, key)
-	c.Assert(err, IsNil)
-	c.Assert(len(resp.Kvs), Equals, 1)
-	c.Assert(string(resp.Kvs[0].Value), Equals, value)
-	
-	// Clean up
-	s.client.Delete(ctx, key)
-}
-
-func (s *QueueSuite) TestQueuedKVDelete(c *C) {
-	qc := NewQueuedEtcdClient(s.client, s.config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, s.config)
-	
-	ctx := context.Background()
-	key := "/test/queue/delete"
-	value := "test-value"
-	
-	// Put directly
-	_, err := s.client.Put(ctx, key, value)
-	c.Assert(err, IsNil)
-	
-	// Delete via queued KV
-	_, err = qkv.Delete(ctx, key)
-	c.Assert(err, IsNil)
-	
-	// Give queue time to process
-	time.Sleep(200 * time.Millisecond)
-	
-	// Verify deletion
-	resp, err := s.client.Get(ctx, key)
-	c.Assert(err, IsNil)
-	c.Assert(len(resp.Kvs), Equals, 0)
-}
-
-func (s *QueueSuite) TestQueuedTransaction(c *C) {
-	qc := NewQueuedEtcdClient(s.client, s.config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, s.config)
-	
-	ctx := context.Background()
-	key1 := "/test/queue/txn1"
-	key2 := "/test/queue/txn2"
-	value1 := "value1"
-	value2 := "value2"
-	
-	// Clean up first
-	s.client.Delete(ctx, key1)
-	s.client.Delete(ctx, key2)
-	
-	// Create transaction
-	txn := qkv.Txn(ctx)
-	txn = txn.If().Then(
-		clientv3.OpPut(key1, value1),
-		clientv3.OpPut(key2, value2),
-	)
-	
-	// Commit via queued transaction
-	_, err := txn.Commit()
-	c.Assert(err, IsNil)
-	
-	// Give queue time to process
-	time.Sleep(200 * time.Millisecond)
-	
-	// Verify both keys exist
-	resp1, err := s.client.Get(ctx, key1)
-	c.Assert(err, IsNil)
-	c.Assert(len(resp1.Kvs), Equals, 1)
-	c.Assert(string(resp1.Kvs[0].Value), Equals, value1)
-	
-	resp2, err := s.client.Get(ctx, key2)
-	c.Assert(err, IsNil)
-	c.Assert(len(resp2.Kvs), Equals, 1)
-	c.Assert(string(resp2.Kvs[0].Value), Equals, value2)
-	
-	// Clean up
-	s.client.Delete(ctx, key1)
-	s.client.Delete(ctx, key2)
-}
-
-func (s *QueueSuite) TestRateLimiting(c *C) {
-	// Configure very low rate limit
-	config := &QueueConfig{
-		Enabled:         true,
-		WriteQueueSize:  100,
-		MaxWritesPerSec: 5, // Only 5 writes per second
-		BatchMaxSize:    10,
-		BatchMaxWait:    10 * time.Millisecond,
-	}
-	
-	qc := NewQueuedEtcdClient(s.client, config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, config)
-	
-	ctx := context.Background()
-	
-	// Time 10 operations
-	start := time.Now()
-	keys := make([]string, 10)
-	for i := 0; i < 10; i++ {
-		key := fmt.Sprintf("/test/queue/rate/%d", i)
-		keys[i] = key
-		_, err := qkv.Put(ctx, key, "value")
-		c.Assert(err, IsNil)
-	}
-	
-	// Clean up after test
-	defer func() {
-		for _, key := range keys {
-			s.client.Delete(ctx, key)
-		}
-	}()
-	
-	// Give queue time to process all
-	time.Sleep(3 * time.Second)
-	
-	// With rate limit of 5/sec, 10 operations should take at least 2 seconds
-	elapsed := time.Since(start)
-	c.Assert(elapsed >= 2*time.Second, Equals, true, Commentf("Operations completed too fast: %v", elapsed))
-}
-
-func (s *QueueSuite) TestConcurrentWrites(c *C) {
-	qc := NewQueuedEtcdClient(s.client, s.config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, s.config)
-	
-	ctx := context.Background()
-	var wg sync.WaitGroup
-	numWriters := 20
-	writesPerWriter := 5
-	
-	var successCount int32
-	
-	// Launch concurrent writers
-	for i := 0; i < numWriters; i++ {
-		wg.Add(1)
-		go func(writerID int) {
-			defer wg.Done()
-			
-			for j := 0; j < writesPerWriter; j++ {
-				key := fmt.Sprintf("/test/queue/concurrent/%d/%d", writerID, j)
-				value := "value"
-				
-				_, err := qkv.Put(ctx, key, value)
-				if err == nil {
-					atomic.AddInt32(&successCount, 1)
-				} else {
-					c.Logf("Write failed: %v", err)
-				}
-				
-				// Clean up immediately
-				if err == nil {
-					s.client.Delete(ctx, key)
-				}
-			}
-		}(i)
-	}
-	
-	// Wait for all writers
-	wg.Wait()
-	
-	// Give queue time to process remaining
-	time.Sleep(2 * time.Second)
-	
-	// All writes should succeed
-	c.Assert(int(successCount), Equals, numWriters*writesPerWriter)
-}
-
-func (s *QueueSuite) TestQueueOverflow(c *C) {
-	c.Skip("Test has blocking issues when queue is full")
-	// This test verifies that when the queue is full, operations don't block indefinitely
-	// Instead, with a small queue, we expect the queue to process items quickly
-	config := &QueueConfig{
-		Enabled:         true,
-		WriteQueueSize:  10, // Small queue but not too small
-		MaxWritesPerSec: 100, // Fast processing
-		BatchMaxSize:    10,
-		BatchMaxWait:    10 * time.Millisecond,
-	}
-	
-	qc := NewQueuedEtcdClient(s.client, config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, config)
-	
-	ctx := context.Background()
-	var wg sync.WaitGroup
-	errors := make(chan error, 20)
-	
-	// Launch 20 concurrent writers
-	for i := 0; i < 20; i++ {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			key := fmt.Sprintf("/test/queue/overflow/%d", idx)
-			_, err := qkv.Put(ctx, key, "value")
-			if err != nil {
-				errors <- err
-			}
-		}(i)
-	}
-	
-	// Wait for all writers to complete
-	wg.Wait()
-	close(errors)
-	
-	// Check for errors
-	for err := range errors {
-		c.Fatalf("Queue operation failed: %v", err)
-	}
-	
-	// Give queue time to finish processing
-	time.Sleep(500 * time.Millisecond)
-}
-
-func (s *QueueSuite) TestDisabledQueue(c *C) {
-	// Create disabled queue
-	config := &QueueConfig{
-		Enabled: false,
-	}
-	
-	qc := NewQueuedEtcdClient(s.client, config)
-	defer qc.Close()
-	
-	qkv := NewQueuedKV(s.client.KV, qc.writeQueue, config)
-	
-	ctx := context.Background()
-	key := "/test/queue/disabled"
-	value := "test-value"
-	
-	// Clean up first
-	s.client.Delete(ctx, key)
-	
-	// Put should go directly to etcd
-	start := time.Now()
-	_, err := qkv.Put(ctx, key, value)
-	c.Assert(err, IsNil)
-	elapsed := time.Since(start)
-	
-	// Should be fast (no queueing)
-	c.Assert(elapsed < 100*time.Millisecond, Equals, true)
-	
-	// Verify immediately
-	resp, err := s.client.Get(ctx, key)
-	c.Assert(err, IsNil)
-	c.Assert(len(resp.Kvs), Equals, 1)
-	c.Assert(string(resp.Kvs[0].Value), Equals, value)
-	
-	// Clean up
-	s.client.Delete(ctx, key)
-}
-
-// TestIntegrationWithStorage tests the queue with actual EtcDStorage
-func (s *QueueSuite) TestIntegrationWithStorage(c *C) {
-	// Create storage with queue
-	storage, err := NewDBWithQueue("localhost:2379", s.config)
-	c.Assert(err, IsNil)
-	defer storage.Close()
-	
-	etcdStorage := storage.(*EtcDStorage)
-	c.Assert(etcdStorage.queuedClient, NotNil)
-	c.Assert(etcdStorage.queuedKV, NotNil)
-	
-	// Test Put/Get operations
-	key := []byte("test-integration-key")
-	value := []byte("test-integration-value")
-	
-	err = etcdStorage.Put(key, value)
-	c.Assert(err, IsNil)
-	
-	// Give queue time to process
-	time.Sleep(200 * time.Millisecond)
-	
-	retrieved, err := etcdStorage.Get(key)
-	c.Assert(err, IsNil)
-	c.Assert(retrieved, DeepEquals, value)
-	
-	// Test Delete
-	err = etcdStorage.Delete(key)
-	c.Assert(err, IsNil)
-	
-	// Give queue time to process
-	time.Sleep(200 * time.Millisecond)
-	
-	retrieved, err = etcdStorage.Get(key)
-	c.Assert(err, IsNil)
-	c.Assert(retrieved, IsNil)
-}

database/etcddb/queue_unit_test.go

@@ -1,51 +0,0 @@
-package etcddb
-
-import (
-	"testing"
-	"time"
-)
-
-func TestQueueConfigDefaults(t *testing.T) {
-	config := &QueueConfig{
-		Enabled: true,
-	}
-
-	// Test default values
-	if config.WriteQueueSize == 0 {
-		config.WriteQueueSize = 1000
-	}
-	if config.MaxWritesPerSec == 0 {
-		config.MaxWritesPerSec = 100
-	}
-	if config.BatchMaxSize == 0 {
-		config.BatchMaxSize = 50
-	}
-	if config.BatchMaxWait == 0 {
-		config.BatchMaxWait = 10 * time.Millisecond
-	}
-
-	// Verify defaults
-	if config.WriteQueueSize != 1000 {
-		t.Errorf("Expected default WriteQueueSize to be 1000, got %d", config.WriteQueueSize)
-	}
-	if config.MaxWritesPerSec != 100 {
-		t.Errorf("Expected default MaxWritesPerSec to be 100, got %d", config.MaxWritesPerSec)
-	}
-	if config.BatchMaxSize != 50 {
-		t.Errorf("Expected default BatchMaxSize to be 50, got %d", config.BatchMaxSize)
-	}
-	if config.BatchMaxWait != 10*time.Millisecond {
-		t.Errorf("Expected default BatchMaxWait to be 10ms, got %v", config.BatchMaxWait)
-	}
-}
-
-func TestQueueConfigDisabled(t *testing.T) {
-	config := &QueueConfig{
-		Enabled: false,
-	}
-
-	if config.Enabled {
-		t.Error("Expected queue to be disabled")
-	}
-}
-

database/etcddb/storage.go

@@ -1,34 +1,26 @@
 package etcddb
 
 import (
-	"context"
 	"fmt"
-	"strings"
-	"time"
 
 	"github.com/aptly-dev/aptly/database"
 	"github.com/google/uuid"
-	"github.com/rs/zerolog/log"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
 
 type EtcDStorage struct {
-	url          string
-	db           *clientv3.Client
-	queuedClient *QueuedEtcdClient
-	queuedKV     *QueuedKV
-	tmpPrefix    string // prefix for temporary DBs
+	url       string
+	db        *clientv3.Client
+	tmpPrefix string // prefix for temporary DBs
 }
 
 // CreateTemporary creates new DB of the same type in temp dir
 func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
 	tmp := uuid.NewString()
 	return &EtcDStorage{
-		url:          s.url,
-		db:           s.db,
-		queuedClient: s.queuedClient,
-		queuedKV:     s.queuedKV,
-		tmpPrefix:    tmp,
+		url:       s.url,
+		db:        s.db,
+		tmpPrefix: tmp,
 	}, nil
 }
 
@@ -39,70 +31,11 @@ func (s *EtcDStorage) applyPrefix(key []byte) []byte {
 	return key
 }
 
-// getContext returns a context with timeout for etcd operations
-func (s *EtcDStorage) getContext() (context.Context, context.CancelFunc) {
-	return context.WithTimeout(context.Background(), DefaultTimeout)
-}
-
-// isTemporary checks if error is temporary and can be retried
-func isTemporary(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	// Check for context deadline exceeded
-	if err == context.DeadlineExceeded {
-		return true
-	}
-
-	// Check for etcd specific temporary errors
-	switch err {
-	case clientv3.ErrNoAvailableEndpoints:
-		return true
-	default:
-		// Check if error string contains temporary indicators
-		errStr := err.Error()
-		return strings.Contains(errStr, "temporary") ||
-			strings.Contains(errStr, "timeout") ||
-			strings.Contains(errStr, "unavailable") ||
-			strings.Contains(errStr, "connection refused")
-	}
-}
-
 // Get key value from etcd
 func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
 	realKey := s.applyPrefix(key)
-
-	var getResp *clientv3.GetResponse
-	maxRetries := 3
-
-	for i := 0; i < maxRetries; i++ {
-		ctx, cancel := s.getContext()
-		if s.queuedKV != nil {
-			getResp, err = s.queuedKV.Get(ctx, string(realKey))
-		} else {
-			getResp, err = s.db.Get(ctx, string(realKey))
-		}
-		cancel()
-
-		if err == nil {
-			break
-		}
-
-		// Only retry on temporary errors and not on last attempt
-		if i < maxRetries-1 && isTemporary(err) {
-			backoff := time.Duration(i+1) * 100 * time.Millisecond
-			log.Warn().
-				Err(err).
-				Str("key", string(realKey)).
-				Int("attempt", i+1).
-				Dur("backoff", backoff).
-				Msg("etcd: get failed, retrying")
-			time.Sleep(backoff)
-			continue
-		}
-
-		log.Error().Err(err).Str("key", string(realKey)).Msg("etcd: get failed")
+	getResp, err := s.db.Get(Ctx, string(realKey))
+	if err != nil {
 		return
 	}
 	for _, kv := range getResp.Kvs {
@@ -119,17 +52,8 @@ func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
 // Put saves key to etcd, if key has the same value in DB already, it is not saved
 func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
 	realKey := s.applyPrefix(key)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	if s.queuedKV != nil {
-		_, err = s.queuedKV.Put(ctx, string(realKey), string(value))
-	} else {
-		_, err = s.db.Put(ctx, string(realKey), string(value))
-	}
+	_, err = s.db.Put(Ctx, string(realKey), string(value))
 	if err != nil {
-		log.Error().Err(err).Str("key", string(realKey)).Msg("etcd: put failed")
 		return
 	}
 	return
@@ -138,17 +62,8 @@ func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
 // Delete removes key from etcd
 func (s *EtcDStorage) Delete(key []byte) (err error) {
 	realKey := s.applyPrefix(key)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	if s.queuedKV != nil {
-		_, err = s.queuedKV.Delete(ctx, string(realKey))
-	} else {
-		_, err = s.db.Delete(ctx, string(realKey))
-	}
+	_, err = s.db.Delete(Ctx, string(realKey))
 	if err != nil {
-		log.Error().Err(err).Str("key", string(realKey)).Msg("etcd: delete failed")
 		return
 	}
 	return
@@ -158,19 +73,8 @@ func (s *EtcDStorage) Delete(key []byte) (err error) {
 func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
 	realPrefix := s.applyPrefix(prefix)
 	result := make([][]byte, 0, 20)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	var getResp *clientv3.GetResponse
-	var err error
-	if s.queuedKV != nil {
-		getResp, err = s.queuedKV.Get(ctx, string(realPrefix), clientv3.WithPrefix())
-	} else {
-		getResp, err = s.db.Get(ctx, string(realPrefix), clientv3.WithPrefix())
-	}
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
 	if err != nil {
-		log.Error().Err(err).Str("prefix", string(realPrefix)).Msg("etcd: keys by prefix failed")
 		return nil
 	}
 	for _, ev := range getResp.Kvs {
@@ -186,13 +90,8 @@ func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
 func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
 	realPrefix := s.applyPrefix(prefix)
 	result := make([][]byte, 0, 20)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	getResp, err := s.db.Get(ctx, string(realPrefix), clientv3.WithPrefix())
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
 	if err != nil {
-		log.Error().Err(err).Str("prefix", string(realPrefix)).Msg("etcd: fetch by prefix failed")
 		return nil
 	}
 	for _, kv := range getResp.Kvs {
@@ -207,13 +106,8 @@ func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
 // HasPrefix checks whether it can find any key with given prefix and returns true if one exists
 func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
 	realPrefix := s.applyPrefix(prefix)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	getResp, err := s.db.Get(ctx, string(realPrefix), clientv3.WithPrefix())
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
 	if err != nil {
-		log.Error().Err(err).Str("prefix", string(realPrefix)).Msg("etcd: has prefix failed")
 		return false
 	}
 	return getResp.Count > 0
@@ -223,13 +117,8 @@ func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
 // StorageProcessor on key value pair
 func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
 	realPrefix := s.applyPrefix(prefix)
-
-	ctx, cancel := s.getContext()
-	defer cancel()
-
-	getResp, err := s.db.Get(ctx, string(realPrefix), clientv3.WithPrefix())
+	getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
 	if err != nil {
-		log.Error().Err(err).Str("prefix", string(realPrefix)).Msg("etcd: process by prefix failed")
 		return err
 	}
 
@@ -248,16 +137,6 @@ func (s *EtcDStorage) Close() error {
 	if len(s.tmpPrefix) != 0 {
 		return nil
 	}
-	
-	if s.queuedClient != nil {
-		// Close queued client first
-		if err := s.queuedClient.Close(); err != nil {
-			log.Warn().Err(err).Msg("etcd: error closing queued client")
-		}
-		s.queuedClient = nil
-		s.queuedKV = nil
-	}
-	
 	if s.db == nil {
 		return nil
 	}
@@ -303,15 +182,12 @@ func (s *EtcDStorage) CompactDB() error {
 // Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys)
 func (s *EtcDStorage) Drop() error {
 	if len(s.tmpPrefix) != 0 {
-		ctx, cancel := s.getContext()
-		defer cancel()
-
-		getResp, err := s.db.Get(ctx, s.tmpPrefix, clientv3.WithPrefix())
+		getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix())
 		if err != nil {
 			return nil
 		}
 		for _, kv := range getResp.Kvs {
-			_, err = s.db.Delete(ctx, string(kv.Key))
+			_, err = s.db.Delete(Ctx, string(kv.Key))
 			if err != nil {
 				return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key)
 			}

database/etcddb/storage_test.go

@@ -1,110 +0,0 @@
-package etcddb
-
-import (
-	"context"
-	"os"
-	"testing"
-	"time"
-
-	. "gopkg.in/check.v1"
-)
-
-type StorageSuite struct{}
-
-var _ = Suite(&StorageSuite{})
-
-func Test(t *testing.T) { TestingT(t) }
-
-func (s *StorageSuite) TestGetContext(c *C) {
-	storage := &EtcDStorage{}
-
-	// Test default timeout
-	ctx, cancel := storage.getContext()
-	defer cancel()
-
-	deadline, ok := ctx.Deadline()
-	c.Assert(ok, Equals, true)
-
-	// Should have a deadline set
-	remaining := time.Until(deadline)
-	c.Assert(remaining > 0, Equals, true)
-	c.Assert(remaining <= DefaultTimeout, Equals, true)
-}
-
-func (s *StorageSuite) TestDefaultTimeout(c *C) {
-	// Default should be 60 seconds
-	c.Assert(DefaultTimeout, Equals, 60*time.Second)
-}
-
-func (s *StorageSuite) TestEnvironmentVariables(c *C) {
-	// Save original values
-	originalTimeout := os.Getenv("APTLY_ETCD_TIMEOUT")
-	originalDialTimeout := os.Getenv("APTLY_ETCD_DIAL_TIMEOUT")
-	originalKeepAlive := os.Getenv("APTLY_ETCD_KEEPALIVE")
-	originalMaxMsg := os.Getenv("APTLY_ETCD_MAX_MSG_SIZE")
-
-	defer func() {
-		// Restore original values
-		os.Setenv("APTLY_ETCD_TIMEOUT", originalTimeout)
-		os.Setenv("APTLY_ETCD_DIAL_TIMEOUT", originalDialTimeout)
-		os.Setenv("APTLY_ETCD_KEEPALIVE", originalKeepAlive)
-		os.Setenv("APTLY_ETCD_MAX_MSG_SIZE", originalMaxMsg)
-	}()
-
-	// Test valid timeout
-	os.Setenv("APTLY_ETCD_TIMEOUT", "30s")
-	// Would need to reinitialize to test, but we can't easily do that
-	// This test mainly ensures the env vars are recognized
-
-	// Test invalid timeout (should use default)
-	os.Setenv("APTLY_ETCD_TIMEOUT", "invalid")
-	timeout := os.Getenv("APTLY_ETCD_TIMEOUT")
-	c.Assert(timeout, Equals, "invalid")
-}
-
-func (s *StorageSuite) TestIsTemporary(c *C) {
-	// Test nil error
-	c.Assert(isTemporary(nil), Equals, false)
-
-	// Test context deadline exceeded
-	c.Assert(isTemporary(context.DeadlineExceeded), Equals, true)
-
-	// Test timeout error
-	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
-	defer cancel()
-	time.Sleep(10 * time.Millisecond)
-	<-ctx.Done()
-	c.Assert(isTemporary(ctx.Err()), Equals, true)
-}
-
-func (s *StorageSuite) TestApplyPrefix(c *C) {
-	// Test without temp prefix
-	storage := &EtcDStorage{}
-	key := []byte("test-key")
-	result := storage.applyPrefix(key)
-	c.Assert(result, DeepEquals, key)
-
-	// Test with temp prefix
-	storage.tmpPrefix = "temp123"
-	result = storage.applyPrefix(key)
-	expected := append([]byte("temp123/"), key...)
-	c.Assert(result, DeepEquals, expected)
-}
-
-// Mock test for retry logic
-func (s *StorageSuite) TestGetRetryLogic(c *C) {
-	// This would require mocking etcd client, which is complex
-	// The test verifies the retry logic exists and compiles
-	// In production, this would be tested with integration tests
-
-	// Verify retry count
-	maxRetries := 3
-	c.Assert(maxRetries, Equals, 3)
-
-	// Verify backoff calculation
-	for i := 0; i < maxRetries; i++ {
-		backoff := time.Duration(i+1) * 100 * time.Millisecond
-		c.Assert(backoff >= 100*time.Millisecond, Equals, true)
-		c.Assert(backoff <= 300*time.Millisecond, Equals, true)
-	}
-}

database/etcddb/transaction.go

@@ -1,8 +1,6 @@
 package etcddb
 
 import (
-	"context"
-
 	"github.com/aptly-dev/aptly/database"
 	clientv3 "go.etcd.io/etcd/client/v3"
 )
@@ -48,9 +46,7 @@ func (t *transaction) Commit() (err error) {
 
 	batchSize := 128
 	for i := 0; i < len(t.ops); i += batchSize {
-		ctx, cancel := context.WithTimeout(context.Background(), DefaultTimeout)
-		defer cancel()
-		txn := kv.Txn(ctx)
+		txn := kv.Txn(Ctx)
 		end := i + batchSize
 		if end > len(t.ops) {
 			end = len(t.ops)

database/goleveldb/database_extended_test.go

@@ -1,519 +0,0 @@
-package goleveldb_test
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-
-	. "gopkg.in/check.v1"
-
-	"github.com/aptly-dev/aptly/database"
-	"github.com/aptly-dev/aptly/database/goleveldb"
-)
-
-type ExtendedLevelDBSuite struct {
-	tempDir string
-}
-
-var _ = Suite(&ExtendedLevelDBSuite{})
-
-func (s *ExtendedLevelDBSuite) SetUpTest(c *C) {
-	s.tempDir = c.MkDir()
-}
-
-func (s *ExtendedLevelDBSuite) TestNewDB(c *C) {
-	// Test NewDB function
-	dbPath := filepath.Join(s.tempDir, "test-db")
-
-	db, err := goleveldb.NewDB(dbPath)
-	c.Check(err, IsNil)
-	c.Check(db, NotNil)
-
-	// DB should not be open yet
-	_, err = db.Get([]byte("test"))
-	c.Check(err, NotNil) // Should error because DB is not open
-
-	// Open the database
-	err = db.Open()
-	c.Check(err, IsNil)
-
-	// Now should work
-	_, err = db.Get([]byte("test"))
-	c.Check(err, Equals, database.ErrNotFound) // Key not found but no open error
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestNewOpenDB(c *C) {
-	// Test NewOpenDB function
-	dbPath := filepath.Join(s.tempDir, "test-open-db")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-	c.Check(db, NotNil)
-
-	// DB should be open and ready to use
-	_, err = db.Get([]byte("test"))
-	c.Check(err, Equals, database.ErrNotFound) // Key not found but no open error
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestRecoverDBError(c *C) {
-	// Test RecoverDB with invalid path
-	invalidPath := "/invalid/nonexistent/path"
-
-	err := goleveldb.RecoverDB(invalidPath)
-	c.Check(err, NotNil) // Should error with invalid path
-}
-
-func (s *ExtendedLevelDBSuite) TestRecoverDBValidPath(c *C) {
-	// Test RecoverDB with valid database
-	dbPath := filepath.Join(s.tempDir, "recover-test")
-
-	// First create a database
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Add some data
-	err = db.Put([]byte("key1"), []byte("value1"))
-	c.Check(err, IsNil)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-
-	// Now recover it
-	err = goleveldb.RecoverDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Verify data is still there after recovery
-	db2, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	value, err := db2.Get([]byte("key1"))
-	c.Check(err, IsNil)
-	c.Check(value, DeepEquals, []byte("value1"))
-
-	err = db2.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestCreateTemporaryError(c *C) {
-	// Test CreateTemporary with limited permissions (if possible)
-	dbPath := filepath.Join(s.tempDir, "test-temp")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	tempDB, err := db.CreateTemporary()
-	c.Check(err, IsNil)
-	c.Check(tempDB, NotNil)
-
-	// Temporary DB should be usable
-	err = tempDB.Put([]byte("temp-key"), []byte("temp-value"))
-	c.Check(err, IsNil)
-
-	value, err := tempDB.Get([]byte("temp-key"))
-	c.Check(err, IsNil)
-	c.Check(value, DeepEquals, []byte("temp-value"))
-
-	err = tempDB.Close()
-	c.Check(err, IsNil)
-
-	err = tempDB.Drop()
-	c.Check(err, IsNil)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestStoragePutOptimization(c *C) {
-	// Test Put optimization (doesn't save if value is same)
-	dbPath := filepath.Join(s.tempDir, "put-optimization")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	key := []byte("optimization-key")
-	value := []byte("same-value")
-
-	// First put
-	err = db.Put(key, value)
-	c.Check(err, IsNil)
-
-	// Second put with same value (should be optimized)
-	err = db.Put(key, value)
-	c.Check(err, IsNil)
-
-	// Third put with different value
-	newValue := []byte("different-value")
-	err = db.Put(key, newValue)
-	c.Check(err, IsNil)
-
-	// Verify final value
-	result, err := db.Get(key)
-	c.Check(err, IsNil)
-	c.Check(result, DeepEquals, newValue)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestStorageCloseMultiple(c *C) {
-	// Test calling Close multiple times
-	dbPath := filepath.Join(s.tempDir, "close-multiple")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// First close should work
-	err = db.Close()
-	c.Check(err, IsNil)
-
-	// Second close should not error
-	err = db.Close()
-	c.Check(err, IsNil)
-
-	// Third close should not error
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestStorageOpenMultiple(c *C) {
-	// Test calling Open multiple times
-	dbPath := filepath.Join(s.tempDir, "open-multiple")
-
-	db, err := goleveldb.NewDB(dbPath)
-	c.Check(err, IsNil)
-
-	// First open should work
-	err = db.Open()
-	c.Check(err, IsNil)
-
-	// Second open should not error (already open)
-	err = db.Open()
-	c.Check(err, IsNil)
-
-	// Should still be functional
-	err = db.Put([]byte("test"), []byte("value"))
-	c.Check(err, IsNil)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestStorageDropError(c *C) {
-	// Test Drop when database is still open
-	dbPath := filepath.Join(s.tempDir, "drop-error")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Try to drop while DB is open (should error)
-	err = db.Drop()
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "DB is still open")
-
-	// Close and then drop should work
-	err = db.Close()
-	c.Check(err, IsNil)
-
-	err = db.Drop()
-	c.Check(err, IsNil)
-
-	// Verify directory is gone
-	_, err = os.Stat(dbPath)
-	c.Check(os.IsNotExist(err), Equals, true)
-}
-
-func (s *ExtendedLevelDBSuite) TestTransactionInterface(c *C) {
-	// Test transaction functionality
-	dbPath := filepath.Join(s.tempDir, "transaction-test")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Create transaction
-	tx, err := db.OpenTransaction()
-	c.Check(err, IsNil)
-	c.Check(tx, NotNil)
-
-	// Test transaction operations
-	key := []byte("tx-key")
-	value := []byte("tx-value")
-
-	err = tx.Put(key, value)
-	c.Check(err, IsNil)
-
-	// Value should not be visible outside transaction yet
-	_, err = db.Get(key)
-	c.Check(err, Equals, database.ErrNotFound)
-
-	// But should be visible within transaction
-	txValue, err := tx.Get(key)
-	c.Check(err, IsNil)
-	c.Check(txValue, DeepEquals, value)
-
-	// Commit transaction
-	err = tx.Commit()
-	c.Check(err, IsNil)
-
-	// Now value should be visible
-	finalValue, err := db.Get(key)
-	c.Check(err, IsNil)
-	c.Check(finalValue, DeepEquals, value)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestTransactionDiscard(c *C) {
-	// Test transaction discard functionality
-	dbPath := filepath.Join(s.tempDir, "transaction-discard")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Create transaction
-	tx, err := db.OpenTransaction()
-	c.Check(err, IsNil)
-
-	key := []byte("discard-key")
-	value := []byte("discard-value")
-
-	err = tx.Put(key, value)
-	c.Check(err, IsNil)
-
-	// Discard transaction
-	tx.Discard()
-
-	// Value should not be visible
-	_, err = db.Get(key)
-	c.Check(err, Equals, database.ErrNotFound)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestProcessByPrefixError(c *C) {
-	// Test ProcessByPrefix with processor that returns error
-	dbPath := filepath.Join(s.tempDir, "process-error")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Add some data
-	prefix := []byte("error-")
-	err = db.Put(append(prefix, []byte("key1")...), []byte("value1"))
-	c.Check(err, IsNil)
-	err = db.Put(append(prefix, []byte("key2")...), []byte("value2"))
-	c.Check(err, IsNil)
-
-	// Process with error-returning function
-	testError := errors.New("processing error")
-	processedCount := 0
-
-	err = db.ProcessByPrefix(prefix, func(key, value []byte) error {
-		processedCount++
-		if processedCount == 1 {
-			return testError
-		}
-		return nil
-	})
-
-	c.Check(err, Equals, testError)
-	c.Check(processedCount, Equals, 1) // Should stop at first error
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestPrefixOperationsEmptyDB(c *C) {
-	// Test prefix operations on empty database
-	dbPath := filepath.Join(s.tempDir, "empty-prefix")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	prefix := []byte("empty")
-
-	// All prefix operations should return empty results
-	c.Check(db.HasPrefix(prefix), Equals, false)
-	c.Check(db.KeysByPrefix(prefix), DeepEquals, [][]byte{})
-	c.Check(db.FetchByPrefix(prefix), DeepEquals, [][]byte{})
-
-	processedCount := 0
-	err = db.ProcessByPrefix(prefix, func(key, value []byte) error {
-		processedCount++
-		return nil
-	})
-	c.Check(err, IsNil)
-	c.Check(processedCount, Equals, 0)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestBatchOperations(c *C) {
-	// Test batch operations in detail
-	dbPath := filepath.Join(s.tempDir, "batch-ops")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Create batch
-	batch := db.CreateBatch()
-	c.Check(batch, NotNil)
-
-	// Add multiple operations to batch
-	keys := [][]byte{
-		[]byte("batch-key-1"),
-		[]byte("batch-key-2"),
-		[]byte("batch-key-3"),
-	}
-	values := [][]byte{
-		[]byte("batch-value-1"),
-		[]byte("batch-value-2"),
-		[]byte("batch-value-3"),
-	}
-
-	for i, key := range keys {
-		err = batch.Put(key, values[i])
-		c.Check(err, IsNil)
-	}
-
-	// Values should not be visible before Write
-	for _, key := range keys {
-		_, err = db.Get(key)
-		c.Check(err, Equals, database.ErrNotFound)
-	}
-
-	// Write batch
-	err = batch.Write()
-	c.Check(err, IsNil)
-
-	// Now all values should be visible
-	for i, key := range keys {
-		value, err := db.Get(key)
-		c.Check(err, IsNil)
-		c.Check(value, DeepEquals, values[i])
-	}
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestIteratorEdgeCases(c *C) {
-	// Test iterator edge cases in prefix operations
-	dbPath := filepath.Join(s.tempDir, "iterator-edge")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Add data with similar but different prefixes
-	prefixes := [][]byte{
-		[]byte("test"),
-		[]byte("test-"),
-		[]byte("test-a"),
-		[]byte("test-ab"),
-		[]byte("testing"),
-		[]byte("totally-different"),
-	}
-
-	for i, prefix := range prefixes {
-		key := append(prefix, []byte("key")...)
-		value := []byte{byte(i)}
-		err = db.Put(key, value)
-		c.Check(err, IsNil)
-	}
-
-	// Test exact prefix matching
-	targetPrefix := []byte("test-")
-	keys := db.KeysByPrefix(targetPrefix)
-	values := db.FetchByPrefix(targetPrefix)
-
-	// Should only match keys that start with "test-"
-	expectedCount := 0
-	for _, prefix := range prefixes {
-		testKey := append(prefix, []byte("key")...)
-		if len(testKey) >= len(targetPrefix) {
-			if string(testKey[:len(targetPrefix)]) == string(targetPrefix) {
-				expectedCount++
-			}
-		}
-	}
-
-	c.Check(len(keys), Equals, expectedCount)
-	c.Check(len(values), Equals, expectedCount)
-
-	err = db.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestCompactDBError(c *C) {
-	// Test CompactDB on closed database
-	dbPath := filepath.Join(s.tempDir, "compact-error")
-
-	db, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-
-	// Close database
-	err = db.Close()
-	c.Check(err, IsNil)
-
-	// CompactDB should error on closed database
-	err = db.CompactDB()
-	c.Check(err, NotNil)
-}
-
-func (s *ExtendedLevelDBSuite) TestInterface(c *C) {
-	// Test that storage implements database.Storage interface
-	dbPath := filepath.Join(s.tempDir, "interface-test")
-
-	var storage database.Storage
-	storage, err := goleveldb.NewOpenDB(dbPath)
-	c.Check(err, IsNil)
-	c.Check(storage, NotNil)
-
-	// Test that all interface methods are available
-	_, err = storage.Get([]byte("test"))
-	c.Check(err, Equals, database.ErrNotFound)
-
-	err = storage.Put([]byte("test"), []byte("value"))
-	c.Check(err, IsNil)
-
-	err = storage.Delete([]byte("test"))
-	c.Check(err, IsNil)
-
-	c.Check(storage.HasPrefix([]byte("test")), Equals, false)
-	c.Check(storage.KeysByPrefix([]byte("test")), DeepEquals, [][]byte{})
-	c.Check(storage.FetchByPrefix([]byte("test")), DeepEquals, [][]byte{})
-
-	err = storage.ProcessByPrefix([]byte("test"), func(k, v []byte) error { return nil })
-	c.Check(err, IsNil)
-
-	batch := storage.CreateBatch()
-	c.Check(batch, NotNil)
-
-	tx, err := storage.OpenTransaction()
-	c.Check(err, IsNil)
-	c.Check(tx, NotNil)
-	tx.Discard()
-
-	temp, err := storage.CreateTemporary()
-	c.Check(err, IsNil)
-	c.Check(temp, NotNil)
-	temp.Close()
-	temp.Drop()
-
-	err = storage.CompactDB()
-	c.Check(err, IsNil)
-
-	err = storage.Close()
-	c.Check(err, IsNil)
-
-	err = storage.Drop()
-	c.Check(err, IsNil)
-}

database/goleveldb/storage.go

@@ -32,9 +32,6 @@ func (s *storage) CreateTemporary() (database.Storage, error) {
 
 // Get key value from database
 func (s *storage) Get(key []byte) ([]byte, error) {
-	if s.db == nil {
-		return nil, errors.New("database not open")
-	}
 	value, err := s.db.Get(key, nil)
 	if err != nil {
 		if err == leveldb.ErrNotFound {
@@ -48,9 +45,6 @@ func (s *storage) Get(key []byte) ([]byte, error) {
 
 // Put saves key to database, if key has the same value in DB already, it is not saved
 func (s *storage) Put(key []byte, value []byte) error {
-	if s.db == nil {
-		return errors.New("database not open")
-	}
 	old, err := s.db.Get(key, nil)
 	if err != nil {
 		if err != leveldb.ErrNotFound {
@@ -66,17 +60,11 @@ func (s *storage) Put(key []byte, value []byte) error {
 
 // Delete removes key from DB
 func (s *storage) Delete(key []byte) error {
-	if s.db == nil {
-		return errors.New("database not open")
-	}
 	return s.db.Delete(key, nil)
 }
 
 // KeysByPrefix returns all keys that start with prefix
 func (s *storage) KeysByPrefix(prefix []byte) [][]byte {
-	if s.db == nil {
-		return nil
-	}
 	result := make([][]byte, 0, 20)
 
 	iterator := s.db.NewIterator(nil, nil)
@@ -94,9 +82,6 @@ func (s *storage) KeysByPrefix(prefix []byte) [][]byte {
 
 // FetchByPrefix returns all values with keys that start with prefix
 func (s *storage) FetchByPrefix(prefix []byte) [][]byte {
-	if s.db == nil {
-		return nil
-	}
 	result := make([][]byte, 0, 20)
 
 	iterator := s.db.NewIterator(nil, nil)
@@ -114,9 +99,6 @@ func (s *storage) FetchByPrefix(prefix []byte) [][]byte {
 
 // HasPrefix checks whether it can find any key with given prefix and returns true if one exists
 func (s *storage) HasPrefix(prefix []byte) bool {
-	if s.db == nil {
-		return false
-	}
 	iterator := s.db.NewIterator(nil, nil)
 	defer iterator.Release()
 	return iterator.Seek(prefix) && bytes.HasPrefix(iterator.Key(), prefix)
@@ -125,9 +107,6 @@ func (s *storage) HasPrefix(prefix []byte) bool {
 // ProcessByPrefix iterates through all entries where key starts with prefix and calls
 // StorageProcessor on key value pair
 func (s *storage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
-	if s.db == nil {
-		return errors.New("database not open")
-	}
 	iterator := s.db.NewIterator(nil, nil)
 	defer iterator.Release()
 
@@ -164,9 +143,6 @@ func (s *storage) Open() error {
 
 // CreateBatch creates a Batch object
 func (s *storage) CreateBatch() database.Batch {
-	if s.db == nil {
-		return nil
-	}
 	return &batch{
 		db: s.db,
 		b:  &leveldb.Batch{},
@@ -175,9 +151,6 @@ func (s *storage) CreateBatch() database.Batch {
 
 // OpenTransaction creates new transaction.
 func (s *storage) OpenTransaction() (database.Transaction, error) {
-	if s.db == nil {
-		return nil, errors.New("database not open")
-	}
 	t, err := s.db.OpenTransaction()
 	if err != nil {
 		return nil, err
@@ -188,9 +161,6 @@ func (s *storage) OpenTransaction() (database.Transaction, error) {
 
 // CompactDB compacts database by merging layers
 func (s *storage) CompactDB() error {
-	if s.db == nil {
-		return errors.New("database not open")
-	}
 	return s.db.CompactRange(util.Range{})
 }
 

database/goleveldb/storage_race_test.go

@@ -1,270 +0,0 @@
-package goleveldb
-
-import (
-	"fmt"
-	"os"
-	"sync"
-	"testing"
-	"time"
-)
-
-// Test for database close race condition
-func TestStorageCloseRace(t *testing.T) {
-	// Create temporary storage
-	tempdir, err := os.MkdirTemp("", "aptly-race-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tempdir)
-
-	db, err := internalOpen(tempdir, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	storage := &storage{db: db, path: tempdir}
-
-	// Put some initial data
-	err = storage.Put([]byte("test-key"), []byte("test-value"))
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	var wg sync.WaitGroup
-	errors := make(chan error, 100)
-	panics := make(chan string, 100)
-
-	// Start multiple goroutines doing database operations
-	for i := 0; i < 10; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panics <- fmt.Sprintf("goroutine %d panicked: %v", id, r)
-				}
-			}()
-
-			// Continuously perform operations
-			for j := 0; j < 100; j++ {
-				// Try Get operation
-				_, err := storage.Get([]byte("test-key"))
-				if err != nil && err.Error() != "database is nil" {
-					errors <- fmt.Errorf("get error in goroutine %d: %v", id, err)
-					return
-				}
-
-				// Try Put operation
-				err = storage.Put([]byte(fmt.Sprintf("key-%d-%d", id, j)), []byte("value"))
-				if err != nil && err.Error() != "database is nil" {
-					errors <- fmt.Errorf("put error in goroutine %d: %v", id, err)
-					return
-				}
-
-				// Small delay to increase race window
-				time.Sleep(time.Microsecond)
-			}
-		}(i)
-	}
-
-	// Start goroutines that close the database
-	for i := 0; i < 5; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panics <- fmt.Sprintf("close goroutine %d panicked: %v", id, r)
-				}
-			}()
-
-			// Wait a bit then close
-			time.Sleep(time.Duration(id*10) * time.Millisecond)
-			err := storage.Close()
-			if err != nil {
-				errors <- fmt.Errorf("close error in goroutine %d: %v", id, err)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	close(errors)
-	close(panics)
-
-	// Check for panics (indicates race condition bug)
-	for panic := range panics {
-		t.Errorf("Race condition caused panic: %s", panic)
-	}
-
-	// Some errors are expected (database closed), but panics are not
-	errorCount := 0
-	for err := range errors {
-		errorCount++
-		if errorCount < 10 { // Only log first few errors
-			t.Logf("Expected error during race: %v", err)
-		}
-	}
-}
-
-// Test concurrent operations vs close
-func TestStorageConcurrentOpsVsClose(t *testing.T) {
-	for attempt := 0; attempt < 5; attempt++ {
-		// Create fresh storage for each attempt
-		tempdir, err := os.MkdirTemp("", "aptly-concurrent-test")
-		if err != nil {
-			t.Fatal(err)
-		}
-		defer os.RemoveAll(tempdir)
-
-		db, err := internalOpen(tempdir, true)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		storage := &storage{db: db, path: tempdir}
-
-		var wg sync.WaitGroup
-		panicked := make(chan bool, 1)
-
-		// Goroutine performing operations
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panicked <- true
-				}
-			}()
-
-			for i := 0; i < 1000; i++ {
-				storage.Get([]byte("key"))
-				storage.Put([]byte("key"), []byte("value"))
-				storage.Delete([]byte("key"))
-			}
-		}()
-
-		// Goroutine closing database
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			time.Sleep(50 * time.Millisecond) // Let operations start
-			storage.Close()
-		}()
-
-		wg.Wait()
-
-		// Check if panic occurred
-		select {
-		case <-panicked:
-			t.Errorf("Attempt %d: Panic occurred during concurrent ops vs close", attempt)
-		default:
-			// No panic - good
-		}
-
-		close(panicked)
-	}
-}
-
-// Test multiple concurrent close attempts
-func TestStorageMultipleClose(t *testing.T) {
-	tempdir, err := os.MkdirTemp("", "aptly-close-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tempdir)
-
-	db, err := internalOpen(tempdir, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	storage := &storage{db: db, path: tempdir}
-
-	var wg sync.WaitGroup
-	panics := make(chan string, 20)
-
-	// Multiple goroutines trying to close
-	for i := 0; i < 20; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panics <- fmt.Sprintf("close %d panicked: %v", id, r)
-				}
-			}()
-
-			err := storage.Close()
-			if err != nil {
-				// Error is ok, panic is not
-				t.Logf("Close %d got error (expected): %v", id, err)
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	close(panics)
-
-	// Check for panics
-	for panic := range panics {
-		t.Errorf("Multiple close caused panic: %s", panic)
-	}
-}
-
-// Test iterator operations during close
-func TestStorageIteratorRace(t *testing.T) {
-	tempdir, err := os.MkdirTemp("", "aptly-iterator-test")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(tempdir)
-
-	db, err := internalOpen(tempdir, true)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	storage := &storage{db: db, path: tempdir}
-
-	// Add some data
-	for i := 0; i < 100; i++ {
-		storage.Put([]byte(fmt.Sprintf("key-%03d", i)), []byte("value"))
-	}
-
-	var wg sync.WaitGroup
-	panics := make(chan string, 10)
-
-	// Goroutines using iterators
-	for i := 0; i < 5; i++ {
-		wg.Add(1)
-		go func(id int) {
-			defer wg.Done()
-			defer func() {
-				if r := recover(); r != nil {
-					panics <- fmt.Sprintf("iterator %d panicked: %v", id, r)
-				}
-			}()
-
-			// Use methods that create iterators
-			storage.KeysByPrefix([]byte("key-"))
-			storage.FetchByPrefix([]byte("key-"))
-			storage.HasPrefix([]byte("key-"))
-		}(i)
-	}
-
-	// Close database while iterators are running
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		time.Sleep(10 * time.Millisecond)
-		storage.Close()
-	}()
-
-	wg.Wait()
-	close(panics)
-
-	// Check for panics
-	for panic := range panics {
-		t.Errorf("Iterator race caused panic: %s", panic)
-	}
-}

database/goleveldb/storage_test.go

@@ -1,121 +0,0 @@
-package goleveldb
-
-import (
-	"github.com/aptly-dev/aptly/database"
-	. "gopkg.in/check.v1"
-)
-
-type LevelDBStorageSuite struct {
-	storage *storage
-	tempDir string
-}
-
-var _ = Suite(&LevelDBStorageSuite{})
-
-func (s *LevelDBStorageSuite) SetUpTest(c *C) {
-	s.tempDir = c.MkDir()
-	s.storage = &storage{
-		path: s.tempDir,
-		db:   nil, // Not opened for unit tests
-	}
-}
-
-func (s *LevelDBStorageSuite) TestCreateTemporary(c *C) {
-	// Test creating temporary storage
-	tempStorage, err := s.storage.CreateTemporary()
-	if err != nil {
-		// Expected to fail without real leveldb setup
-		c.Check(err, NotNil)
-		return
-	}
-
-	c.Check(tempStorage, NotNil)
-	levelStorage, ok := tempStorage.(*storage)
-	c.Check(ok, Equals, true)
-	c.Check(len(levelStorage.path) > 0, Equals, true)
-	c.Check(levelStorage.path, Not(Equals), s.storage.path)
-}
-
-func (s *LevelDBStorageSuite) TestCloseNilDB(c *C) {
-	// Test closing storage with nil DB
-	err := s.storage.Close()
-	c.Check(err, IsNil)
-}
-
-func (s *LevelDBStorageSuite) TestOpenNilDB(c *C) {
-	// Test opening storage - should succeed with valid path
-	err := s.storage.Open()
-	// Should succeed with valid temporary directory
-	c.Check(err, IsNil)
-	// Clean up
-	s.storage.Close()
-}
-
-func (s *LevelDBStorageSuite) TestCreateBatchNilDB(c *C) {
-	// Test creating batch with nil DB
-	batch := s.storage.CreateBatch()
-	c.Check(batch, IsNil)
-}
-
-func (s *LevelDBStorageSuite) TestCompactDB(c *C) {
-	// Test CompactDB with nil DB - should handle gracefully
-	err := s.storage.CompactDB()
-	c.Check(err, NotNil) // Expected to fail with nil DB
-}
-
-func (s *LevelDBStorageSuite) TestDropNilDB(c *C) {
-	// Test dropping storage with nil DB
-	err := s.storage.Drop()
-	c.Check(err, IsNil) // Should succeed (removes directory)
-}
-
-func (s *LevelDBStorageSuite) TestInterfaceCompliance(c *C) {
-	// Test that storage implements database.Storage interface
-	var dbStorage database.Storage = &storage{}
-	c.Check(dbStorage, NotNil)
-}
-
-func (s *LevelDBStorageSuite) TestGetNilDB(c *C) {
-	// Test Get with nil DB - should fail
-	_, err := s.storage.Get([]byte("key"))
-	c.Check(err, NotNil) // Expected to fail with nil DB
-}
-
-// Note: storage does not implement Has method - it uses Get and checks for ErrNotFound
-
-func (s *LevelDBStorageSuite) TestPutNilDB(c *C) {
-	// Test Put with nil DB - should fail
-	err := s.storage.Put([]byte("key"), []byte("value"))
-	c.Check(err, NotNil) // Expected to fail with nil DB
-}
-
-func (s *LevelDBStorageSuite) TestDeleteNilDB(c *C) {
-	// Test Delete with nil DB - should fail
-	err := s.storage.Delete([]byte("key"))
-	c.Check(err, NotNil) // Expected to fail with nil DB
-}
-
-func (s *LevelDBStorageSuite) TestKeysByPrefixNilDB(c *C) {
-	// Test KeysByPrefix with nil DB - should return nil
-	keys := s.storage.KeysByPrefix([]byte("prefix/"))
-	c.Check(keys, IsNil)
-}
-
-func (s *LevelDBStorageSuite) TestFetchByPrefixNilDB(c *C) {
-	// Test FetchByPrefix with nil DB - should return nil
-	values := s.storage.FetchByPrefix([]byte("prefix/"))
-	c.Check(values, IsNil)
-}
-
-func (s *LevelDBStorageSuite) TestHasPrefixNilDB(c *C) {
-	// Test HasPrefix with nil DB - should return false
-	result := s.storage.HasPrefix([]byte("prefix/"))
-	c.Check(result, Equals, false)
-}
-
-func (s *LevelDBStorageSuite) TestProcessByPrefixNilDB(c *C) {
-	// Test ProcessByPrefix with nil DB - should fail
-	processor := func(key, value []byte) error { return nil }
-	err := s.storage.ProcessByPrefix([]byte("prefix/"), processor)
-	c.Check(err, NotNil) // Expected to fail with nil DB
-}

database/goleveldb/transaction_test.go

@@ -1,38 +0,0 @@
-package goleveldb
-
-import (
-	"github.com/aptly-dev/aptly/database"
-	. "gopkg.in/check.v1"
-)
-
-type TransactionSuite struct {
-	storage *storage
-	tempDir string
-}
-
-var _ = Suite(&TransactionSuite{})
-
-func (s *TransactionSuite) SetUpTest(c *C) {
-	s.tempDir = c.MkDir()
-	s.storage = &storage{
-		path: s.tempDir,
-		db:   nil, // Not opened for unit tests
-	}
-}
-
-func (s *TransactionSuite) TestOpenTransactionNilDB(c *C) {
-	// Test opening transaction with nil DB - should fail
-	transaction, err := s.storage.OpenTransaction()
-	c.Check(err, NotNil) // Expected to fail with nil DB
-	c.Check(transaction, IsNil)
-}
-
-func (s *TransactionSuite) TestInterfaceCompliance(c *C) {
-	// Test that storage implements the transaction interface
-	var storageInterface database.Storage = &storage{}
-	c.Check(storageInterface, NotNil)
-
-	// Test that we can call OpenTransaction method
-	_, err := storageInterface.OpenTransaction()
-	c.Check(err, NotNil) // Expected to fail without proper setup
-}

deb/collections_test.go

@@ -1,211 +0,0 @@
-package deb
-
-import (
-	"github.com/aptly-dev/aptly/database"
-	"github.com/aptly-dev/aptly/database/goleveldb"
-	. "gopkg.in/check.v1"
-)
-
-type CollectionsSuite struct {
-	db      database.Storage
-	factory *CollectionFactory
-}
-
-var _ = Suite(&CollectionsSuite{})
-
-func (s *CollectionsSuite) SetUpTest(c *C) {
-	s.db, _ = goleveldb.NewOpenDB(c.MkDir())
-	s.factory = NewCollectionFactory(s.db)
-}
-
-func (s *CollectionsSuite) TearDownTest(c *C) {
-	s.db.Close()
-}
-
-func (s *CollectionsSuite) TestNewCollectionFactory(c *C) {
-	factory := NewCollectionFactory(s.db)
-	c.Check(factory, NotNil)
-	c.Check(factory.db, Equals, s.db)
-	c.Check(factory.Mutex, NotNil)
-}
-
-func (s *CollectionsSuite) TestTemporaryDB(c *C) {
-	tempDB, err := s.factory.TemporaryDB()
-	c.Check(err, IsNil)
-	c.Check(tempDB, NotNil)
-
-	// Clean up
-	tempDB.Close()
-	tempDB.Drop()
-}
-
-func (s *CollectionsSuite) TestPackageCollection(c *C) {
-	// First call creates the collection
-	collection1 := s.factory.PackageCollection()
-	c.Check(collection1, NotNil)
-
-	// Second call returns the same instance
-	collection2 := s.factory.PackageCollection()
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestRemoteRepoCollection(c *C) {
-	// First call creates the collection
-	collection1 := s.factory.RemoteRepoCollection()
-	c.Check(collection1, NotNil)
-
-	// Second call returns the same instance
-	collection2 := s.factory.RemoteRepoCollection()
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestSnapshotCollection(c *C) {
-	// First call creates the collection
-	collection1 := s.factory.SnapshotCollection()
-	c.Check(collection1, NotNil)
-
-	// Second call returns the same instance
-	collection2 := s.factory.SnapshotCollection()
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestLocalRepoCollection(c *C) {
-	// First call creates the collection
-	collection1 := s.factory.LocalRepoCollection()
-	c.Check(collection1, NotNil)
-
-	// Second call returns the same instance
-	collection2 := s.factory.LocalRepoCollection()
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestPublishedRepoCollection(c *C) {
-	// First call creates the collection
-	collection1 := s.factory.PublishedRepoCollection()
-	c.Check(collection1, NotNil)
-
-	// Second call returns the same instance
-	collection2 := s.factory.PublishedRepoCollection()
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestChecksumCollectionWithNilDB(c *C) {
-	// First call with nil DB creates the collection
-	collection1 := s.factory.ChecksumCollection(nil)
-	c.Check(collection1, NotNil)
-
-	// Second call with nil DB returns the same instance
-	collection2 := s.factory.ChecksumCollection(nil)
-	c.Check(collection2, Equals, collection1)
-}
-
-func (s *CollectionsSuite) TestChecksumCollectionWithDB(c *C) {
-	// Create temporary DB
-	tempDB, err := s.factory.TemporaryDB()
-	c.Check(err, IsNil)
-	defer tempDB.Close()
-	defer tempDB.Drop()
-
-	// Call with specific DB creates new collection
-	collection1 := s.factory.ChecksumCollection(tempDB)
-	c.Check(collection1, NotNil)
-
-	// Call with different DB creates different collection
-	collection2 := s.factory.ChecksumCollection(s.db)
-	c.Check(collection2, NotNil)
-	c.Check(collection2, Not(Equals), collection1)
-}
-
-func (s *CollectionsSuite) TestFlush(c *C) {
-	// Create all collections
-	packages := s.factory.PackageCollection()
-	remoteRepos := s.factory.RemoteRepoCollection()
-	snapshots := s.factory.SnapshotCollection()
-	localRepos := s.factory.LocalRepoCollection()
-	publishedRepos := s.factory.PublishedRepoCollection()
-	checksums := s.factory.ChecksumCollection(nil)
-
-	c.Check(packages, NotNil)
-	c.Check(remoteRepos, NotNil)
-	c.Check(snapshots, NotNil)
-	c.Check(localRepos, NotNil)
-	c.Check(publishedRepos, NotNil)
-	c.Check(checksums, NotNil)
-
-	// Flush all collections
-	s.factory.Flush()
-
-	// After flush, new calls should create new instances
-	newPackages := s.factory.PackageCollection()
-	newRemoteRepos := s.factory.RemoteRepoCollection()
-	newSnapshots := s.factory.SnapshotCollection()
-	newLocalRepos := s.factory.LocalRepoCollection()
-	newPublishedRepos := s.factory.PublishedRepoCollection()
-	newChecksums := s.factory.ChecksumCollection(nil)
-
-	c.Check(newPackages, Not(Equals), packages)
-	c.Check(newRemoteRepos, Not(Equals), remoteRepos)
-	c.Check(newSnapshots, Not(Equals), snapshots)
-	c.Check(newLocalRepos, Not(Equals), localRepos)
-	c.Check(newPublishedRepos, Not(Equals), publishedRepos)
-	c.Check(newChecksums, Not(Equals), checksums)
-}
-
-func (s *CollectionsSuite) TestConcurrentAccess(c *C) {
-	// Test that concurrent access to collections works properly
-	done := make(chan bool, 10)
-
-	for i := 0; i < 10; i++ {
-		go func() {
-			// Each goroutine should get the same instances
-			packages := s.factory.PackageCollection()
-			remoteRepos := s.factory.RemoteRepoCollection()
-			snapshots := s.factory.SnapshotCollection()
-			localRepos := s.factory.LocalRepoCollection()
-			publishedRepos := s.factory.PublishedRepoCollection()
-			checksums := s.factory.ChecksumCollection(nil)
-
-			c.Check(packages, NotNil)
-			c.Check(remoteRepos, NotNil)
-			c.Check(snapshots, NotNil)
-			c.Check(localRepos, NotNil)
-			c.Check(publishedRepos, NotNil)
-			c.Check(checksums, NotNil)
-
-			done <- true
-		}()
-	}
-
-	// Wait for all goroutines to complete
-	for i := 0; i < 10; i++ {
-		<-done
-	}
-
-	// Verify that all collections are still accessible
-	packages := s.factory.PackageCollection()
-	c.Check(packages, NotNil)
-}
-
-func (s *CollectionsSuite) TestFlushAndRecreate(c *C) {
-	// Create collections, use them, flush, then recreate
-	originalPackages := s.factory.PackageCollection()
-	c.Check(originalPackages, NotNil)
-
-	// Add a package to test that it exists
-	pkg := NewPackageFromControlFile(packageStanza.Copy())
-	err := originalPackages.Update(pkg)
-	c.Check(err, IsNil)
-
-	// Flush
-	s.factory.Flush()
-
-	// Get new collection
-	newPackages := s.factory.PackageCollection()
-	c.Check(newPackages, NotNil)
-	c.Check(newPackages, Not(Equals), originalPackages)
-
-	// The package should still exist in the database
-	retrievedPkg, err := newPackages.ByKey(pkg.Key(""))
-	c.Check(err, IsNil)
-	c.Check(retrievedPkg.Name, Equals, pkg.Name)
-}

deb/contents_test.go

@@ -1,406 +0,0 @@
-package deb
-
-import (
-	"bytes"
-	"strings"
-
-	"github.com/aptly-dev/aptly/database"
-	. "gopkg.in/check.v1"
-)
-
-type ContentsIndexSuite struct {
-	mockDB *MockStorage
-}
-
-var _ = Suite(&ContentsIndexSuite{})
-
-func (s *ContentsIndexSuite) SetUpTest(c *C) {
-	s.mockDB = &MockStorage{
-		data:     make(map[string][]byte),
-		prefixes: make(map[string]bool),
-	}
-}
-
-func (s *ContentsIndexSuite) TestNewContentsIndex(c *C) {
-	// Test ContentsIndex creation
-	index := NewContentsIndex(s.mockDB)
-
-	c.Check(index, NotNil)
-	c.Check(index.db, Equals, s.mockDB)
-	c.Check(len(index.prefix), Equals, 36) // UUID length
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexEmpty(c *C) {
-	// Test Empty method
-	index := NewContentsIndex(s.mockDB)
-
-	// Should be empty initially
-	c.Check(index.Empty(), Equals, true)
-
-	// Add some data
-	s.mockDB.prefixes[string(index.prefix)] = true
-
-	// Should not be empty now
-	c.Check(index.Empty(), Equals, false)
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexPush(c *C) {
-	// Test Push method
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	qualifiedName := []byte("package_1.0_amd64")
-	contents := []string{
-		"/usr/bin/program",
-		"/usr/share/doc/package/README",
-		"/etc/package.conf",
-	}
-
-	err := index.Push(qualifiedName, contents, writer)
-	c.Check(err, IsNil)
-
-	// Verify data was written
-	c.Check(len(s.mockDB.data), Equals, 3)
-
-	// Check that keys contain the expected format
-	for path := range contents {
-		expectedKey := string(index.prefix) + contents[path] + "\x00" + string(qualifiedName)
-		_, exists := s.mockDB.data[expectedKey]
-		c.Check(exists, Equals, true, Commentf("Missing key for path: %s", contents[path]))
-	}
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexPushError(c *C) {
-	// Test Push method with writer error
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB, shouldError: true}
-
-	qualifiedName := []byte("package_1.0_amd64")
-	contents := []string{"/usr/bin/program"}
-
-	err := index.Push(qualifiedName, contents, writer)
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "mock writer error")
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexWriteTo(c *C) {
-	// Test WriteTo method
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	// Add some packages
-	err := index.Push([]byte("package1_1.0_amd64"), []string{"/usr/bin/prog1", "/usr/share/file1"}, writer)
-	c.Check(err, IsNil)
-
-	err = index.Push([]byte("package2_2.0_amd64"), []string{"/usr/bin/prog2", "/usr/share/file1"}, writer)
-	c.Check(err, IsNil)
-
-	// Set up processor to simulate database iteration
-	s.mockDB.processor = func(prefix []byte, fn database.StorageProcessor) error {
-		// Simulate database keys in sorted order
-		keys := []string{
-			string(prefix) + "/usr/bin/prog1\x00package1_1.0_amd64",
-			string(prefix) + "/usr/bin/prog2\x00package2_2.0_amd64",
-			string(prefix) + "/usr/share/file1\x00package1_1.0_amd64",
-			string(prefix) + "/usr/share/file1\x00package2_2.0_amd64",
-		}
-
-		for _, key := range keys {
-			err := fn([]byte(key), nil)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	var buf bytes.Buffer
-	n, err := index.WriteTo(&buf)
-	c.Check(err, IsNil)
-	c.Check(n, Equals, int64(buf.Len()))
-
-	output := buf.String()
-	lines := strings.Split(strings.TrimSpace(output), "\n")
-
-	// Should have header plus content lines
-	c.Check(len(lines), Equals, 4)
-	c.Check(lines[0], Equals, "FILE LOCATION")
-	c.Check(lines[1], Equals, "/usr/bin/prog1 package1_1.0_amd64")
-	c.Check(lines[2], Equals, "/usr/bin/prog2 package2_2.0_amd64")
-	c.Check(lines[3], Equals, "/usr/share/file1 package1_1.0_amd64,package2_2.0_amd64")
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexWriteToEmpty(c *C) {
-	// Test WriteTo with empty index
-	index := NewContentsIndex(s.mockDB)
-
-	s.mockDB.processor = func(prefix []byte, fn database.StorageProcessor) error {
-		// No entries
-		return nil
-	}
-
-	var buf bytes.Buffer
-	n, err := index.WriteTo(&buf)
-	c.Check(err, IsNil)
-	c.Check(n, Equals, int64(buf.Len()))
-
-	output := buf.String()
-	c.Check(output, Equals, "FILE LOCATION\n")
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexWriteToCorruptedEntry(c *C) {
-	// Test WriteTo with corrupted database entry
-	index := NewContentsIndex(s.mockDB)
-
-	s.mockDB.processor = func(prefix []byte, fn database.StorageProcessor) error {
-		// Corrupted key without null byte separator
-		corruptedKey := string(prefix) + "/usr/bin/prog1package_name"
-		return fn([]byte(corruptedKey), nil)
-	}
-
-	var buf bytes.Buffer
-	_, err := index.WriteTo(&buf)
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Equals, "corrupted index entry")
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexPushMultiplePackages(c *C) {
-	// Test pushing multiple packages
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	packages := []struct {
-		name     string
-		contents []string
-	}{
-		{"package1_1.0_amd64", []string{"/usr/bin/prog1", "/usr/share/doc1"}},
-		{"package2_2.0_amd64", []string{"/usr/bin/prog2", "/usr/share/doc2"}},
-		{"package3_3.0_amd64", []string{"/usr/bin/prog3"}},
-	}
-
-	for _, pkg := range packages {
-		err := index.Push([]byte(pkg.name), pkg.contents, writer)
-		c.Check(err, IsNil, Commentf("Failed to push package: %s", pkg.name))
-	}
-
-	// Verify all entries were written
-	expectedEntries := 2 + 2 + 1 // Total files across all packages
-	c.Check(len(s.mockDB.data), Equals, expectedEntries)
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexPushEmptyContents(c *C) {
-	// Test pushing package with no contents
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	err := index.Push([]byte("empty_package"), []string{}, writer)
-	c.Check(err, IsNil)
-
-	// Should not add any entries
-	c.Check(len(s.mockDB.data), Equals, 0)
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexSpecialCharacters(c *C) {
-	// Test with special characters in paths and package names
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	qualifiedName := []byte("special-package_1.0+build1_amd64")
-	contents := []string{
-		"/usr/bin/prog-with-dashes",
-		"/usr/share/file with spaces",
-		"/etc/config.d/file.conf",
-	}
-
-	err := index.Push(qualifiedName, contents, writer)
-	c.Check(err, IsNil)
-
-	c.Check(len(s.mockDB.data), Equals, 3)
-}
-
-func (s *ContentsIndexSuite) TestContentsIndexBinaryData(c *C) {
-	// Test with binary data in paths (edge case)
-	index := NewContentsIndex(s.mockDB)
-	writer := &MockWriter{storage: s.mockDB}
-
-	// Path with binary data
-	binaryPath := "/usr/bin/prog\x00\xFF\xFE"
-	qualifiedName := []byte("binary_package_1.0_amd64")
-
-	err := index.Push(qualifiedName, []string{binaryPath}, writer)
-	c.Check(err, IsNil)
-
-	c.Check(len(s.mockDB.data), Equals, 1)
-}
-
-// Mock implementations for testing
-type MockStorage struct {
-	data      map[string][]byte
-	prefixes  map[string]bool
-	processor func([]byte, database.StorageProcessor) error
-}
-
-func (m *MockStorage) Get(key []byte) ([]byte, error) {
-	if value, exists := m.data[string(key)]; exists {
-		return value, nil
-	}
-	return nil, database.ErrNotFound
-}
-
-func (m *MockStorage) Put(key, value []byte) error {
-	m.data[string(key)] = value
-	return nil
-}
-
-func (m *MockStorage) Delete(key []byte) error {
-	delete(m.data, string(key))
-	return nil
-}
-
-func (m *MockStorage) HasPrefix(prefix []byte) bool {
-	if exists, ok := m.prefixes[string(prefix)]; ok {
-		return exists
-	}
-
-	// Check if any key has this prefix
-	for key := range m.data {
-		if strings.HasPrefix(key, string(prefix)) {
-			return true
-		}
-	}
-	return false
-}
-
-func (m *MockStorage) ProcessByPrefix(prefix []byte, fn database.StorageProcessor) error {
-	if m.processor != nil {
-		return m.processor(prefix, fn)
-	}
-
-	// Default implementation - process matching keys
-	for key, value := range m.data {
-		if strings.HasPrefix(key, string(prefix)) {
-			err := fn([]byte(key), value)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (m *MockStorage) KeysByPrefix(prefix []byte) [][]byte {
-	var keys [][]byte
-	for key := range m.data {
-		if strings.HasPrefix(key, string(prefix)) {
-			keys = append(keys, []byte(key))
-		}
-	}
-	return keys
-}
-
-func (m *MockStorage) FetchByPrefix(prefix []byte) [][]byte {
-	var values [][]byte
-	for key, value := range m.data {
-		if strings.HasPrefix(key, string(prefix)) {
-			values = append(values, value)
-		}
-	}
-	return values
-}
-
-func (m *MockStorage) Close() error {
-	return nil
-}
-
-func (m *MockStorage) CompactDB() error {
-	return nil
-}
-
-func (m *MockStorage) Drop() error {
-	return nil
-}
-
-func (m *MockStorage) Open() error {
-	return nil
-}
-
-func (m *MockStorage) CreateBatch() database.Batch {
-	return &MockBatch{storage: m}
-}
-
-func (m *MockStorage) OpenTransaction() (database.Transaction, error) {
-	return &MockTransaction{storage: m}, nil
-}
-
-func (m *MockStorage) CreateTemporary() (database.Storage, error) {
-	return &MockStorage{
-		data:     make(map[string][]byte),
-		prefixes: make(map[string]bool),
-	}, nil
-}
-
-type MockBatch struct {
-	storage *MockStorage
-}
-
-func (m *MockBatch) Put(key, value []byte) error {
-	return m.storage.Put(key, value)
-}
-
-func (m *MockBatch) Delete(key []byte) error {
-	return m.storage.Delete(key)
-}
-
-func (m *MockBatch) Write() error {
-	return nil
-}
-
-type MockTransaction struct {
-	storage *MockStorage
-}
-
-func (m *MockTransaction) Get(key []byte) ([]byte, error) {
-	return m.storage.Get(key)
-}
-
-func (m *MockTransaction) Put(key, value []byte) error {
-	return m.storage.Put(key, value)
-}
-
-func (m *MockTransaction) Delete(key []byte) error {
-	return m.storage.Delete(key)
-}
-
-func (m *MockTransaction) Commit() error {
-	return nil
-}
-
-func (m *MockTransaction) Discard() {
-}
-
-type MockWriter struct {
-	storage     *MockStorage
-	shouldError bool
-}
-
-func (m *MockWriter) Put(key, value []byte) error {
-	if m.shouldError {
-		return &MockError{message: "mock writer error"}
-	}
-	return m.storage.Put(key, value)
-}
-
-func (m *MockWriter) Delete(key []byte) error {
-	if m.shouldError {
-		return &MockError{message: "mock writer error"}
-	}
-	return m.storage.Delete(key)
-}
-
-type MockError struct {
-	message string
-}
-
-func (e *MockError) Error() string {
-	return e.message
-}

deb/format.go

@@ -288,8 +288,11 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) {
 			lastField = canonicalCase(parts[0])
 			lastFieldMultiline = isMultilineField(lastField, c.isRelease)
 			if lastFieldMultiline {
-				stanza[lastField] = parts[1]
-				if parts[1] != "" {
+				// Trim trailing whitespace from the inline value so that
+				// "Package-List: " does not add empty line
+				inlineVal := strings.TrimRight(parts[1], " \t")
+				stanza[lastField] = inlineVal
+				if inlineVal != "" {
 					stanza[lastField] += "\n"
 				}
 			} else {

deb/format_test.go

@@ -128,6 +128,35 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) {
 	c.Assert(strings.HasPrefix(str, "Package: "), Equals, true)
 }
 
+// Sources may contain "Package-List: " with a trailing space.
+// That trailing space must not be preserved and re-emitted
+// as a spurious blank continuation line when the stanza is written back out.
+func (s *ControlFileSuite) TestPackageListTrailingSpace(c *C) {
+	input := "Package-List: \n" +
+		" bash deb shells required arch=any\n" +
+		" bash-doc deb doc optional arch=all\n"
+
+	r := NewControlFileReader(bytes.NewBufferString(input), false, false)
+	stanza, err := r.ReadStanza()
+	c.Assert(err, IsNil)
+
+	c.Check(stanza["Package-List"], Equals,
+		" bash deb shells required arch=any\n"+
+			" bash-doc deb doc optional arch=all\n")
+
+	buf := &bytes.Buffer{}
+	w := bufio.NewWriter(buf)
+	err = stanza.Copy().WriteTo(w, true, false, false)
+	c.Assert(err, IsNil)
+	c.Assert(w.Flush(), IsNil)
+
+	written := buf.String()
+	c.Assert(strings.Contains(written, "Package-List:\n \n"), Equals, false,
+		Commentf("spurious blank continuation line found in written output:\n%s", written))
+	c.Assert(strings.Contains(written, "Package-List:\n bash"), Equals, true,
+		Commentf("expected Package-List entries not found in written output:\n%s", written))
+}
+
 func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) {
 	s.reader = bytes.NewBufferString(installerFile)
 	r := NewControlFileReader(s.reader, false, true)

deb/graph_test.go

@@ -1,42 +0,0 @@
-package deb
-
-import (
-	"github.com/aptly-dev/aptly/database/goleveldb"
-	. "gopkg.in/check.v1"
-)
-
-type GraphSuite struct {
-	collectionFactory *CollectionFactory
-}
-
-var _ = Suite(&GraphSuite{})
-
-func (s *GraphSuite) SetUpTest(c *C) {
-	db, _ := goleveldb.NewOpenDB(c.MkDir())
-	s.collectionFactory = NewCollectionFactory(db)
-}
-
-func (s *GraphSuite) TearDownTest(c *C) {
-	// Collections are closed automatically when the test ends
-}
-
-func (s *GraphSuite) TestBuildGraphBasic(c *C) {
-	// Test BuildGraph with default (horizontal) layout
-	graph, err := BuildGraph(s.collectionFactory, "horizontal")
-	c.Check(err, IsNil)
-	c.Check(graph, NotNil)
-}
-
-func (s *GraphSuite) TestBuildGraphVertical(c *C) {
-	// Test BuildGraph with vertical layout
-	graph, err := BuildGraph(s.collectionFactory, "vertical")
-	c.Check(err, IsNil)
-	c.Check(graph, NotNil)
-}
-
-func (s *GraphSuite) TestBuildGraphUnknownLayout(c *C) {
-	// Test BuildGraph with unknown layout (should default to horizontal)
-	graph, err := BuildGraph(s.collectionFactory, "unknown")
-	c.Check(err, IsNil)
-	c.Check(graph, NotNil)
-}

deb/import.go

@@ -92,7 +92,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 		if isSourcePackage {
 			stanza, err = GetControlFileFromDsc(file, verifier)
 
-			if err == nil && stanza != nil {
+			if err == nil {
 				stanza["Package"] = stanza["Source"]
 				delete(stanza, "Source")
 
@@ -100,12 +100,10 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			}
 		} else {
 			stanza, err = GetControlFileFromDeb(file)
-			if err == nil && stanza != nil {
-				if isUdebPackage {
-					p = NewUdebPackageFromControlFile(stanza)
-				} else {
-					p = NewPackageFromControlFile(stanza)
-				}
+			if isUdebPackage {
+				p = NewUdebPackageFromControlFile(stanza)
+			} else {
+				p = NewPackageFromControlFile(stanza)
 			}
 		}
 		if err != nil {
@@ -114,12 +112,6 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
 			continue
 		}
 
-		if p == nil {
-			reporter.Warning("Unable to process package file %s", file)
-			failedFiles = append(failedFiles, file)
-			continue
-		}
-
 		if p.Name == "" {
 			reporter.Warning("Empty package name on %s", file)
 			failedFiles = append(failedFiles, file)

deb/import_test.go

@@ -1,876 +0,0 @@
-package deb
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/database"
-	"github.com/aptly-dev/aptly/pgp"
-	"github.com/aptly-dev/aptly/utils"
-	. "gopkg.in/check.v1"
-)
-
-type ImportSuite struct {
-	tempDir string
-}
-
-var _ = Suite(&ImportSuite{})
-
-func (s *ImportSuite) SetUpTest(c *C) {
-	s.tempDir = c.MkDir()
-}
-
-type MockResultReporter struct {
-	warnings []string
-	added    []string
-	removed  []string
-}
-
-func (m *MockResultReporter) Warning(msg string, a ...interface{}) {
-	m.warnings = append(m.warnings, fmt.Sprintf(msg, a...))
-}
-
-func (m *MockResultReporter) Added(msg string, a ...interface{}) {
-	m.added = append(m.added, fmt.Sprintf(msg, a...))
-}
-
-func (m *MockResultReporter) Removed(msg string, a ...interface{}) {
-	m.removed = append(m.removed, fmt.Sprintf(msg, a...))
-}
-
-func (s *ImportSuite) TestCollectPackageFilesEmpty(c *C) {
-	// Test with empty locations list
-	reporter := &MockResultReporter{}
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{}, reporter)
-
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesNonExistentLocation(c *C) {
-	// Test with non-existent location
-	reporter := &MockResultReporter{}
-	nonExistentPath := filepath.Join(s.tempDir, "nonexistent")
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{nonExistentPath}, reporter)
-
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, nonExistentPath)
-	c.Check(len(reporter.warnings), Equals, 1)
-	c.Check(strings.Contains(reporter.warnings[0], "Unable to process"), Equals, true)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSingleDebFile(c *C) {
-	// Test with single .deb file
-	reporter := &MockResultReporter{}
-	debFile := filepath.Join(s.tempDir, "package.deb")
-
-	// Create dummy .deb file
-	err := ioutil.WriteFile(debFile, []byte("dummy deb content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{debFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 1)
-	c.Check(packageFiles[0], Equals, debFile)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSingleUdebFile(c *C) {
-	// Test with single .udeb file
-	reporter := &MockResultReporter{}
-	udebFile := filepath.Join(s.tempDir, "package.udeb")
-
-	err := ioutil.WriteFile(udebFile, []byte("dummy udeb content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{udebFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 1)
-	c.Check(packageFiles[0], Equals, udebFile)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSingleDscFile(c *C) {
-	// Test with single .dsc file
-	reporter := &MockResultReporter{}
-	dscFile := filepath.Join(s.tempDir, "package.dsc")
-
-	err := ioutil.WriteFile(dscFile, []byte("dummy dsc content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{dscFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 1)
-	c.Check(packageFiles[0], Equals, dscFile)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSingleDdebFile(c *C) {
-	// Test with single .ddeb file
-	reporter := &MockResultReporter{}
-	ddebFile := filepath.Join(s.tempDir, "package.ddeb")
-
-	err := ioutil.WriteFile(ddebFile, []byte("dummy ddeb content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{ddebFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 1)
-	c.Check(packageFiles[0], Equals, ddebFile)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesBuildInfoFile(c *C) {
-	// Test with .buildinfo file
-	reporter := &MockResultReporter{}
-	buildinfoFile := filepath.Join(s.tempDir, "package.buildinfo")
-
-	err := ioutil.WriteFile(buildinfoFile, []byte("dummy buildinfo content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{buildinfoFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 1)
-	c.Check(otherFiles[0], Equals, buildinfoFile)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesUnknownExtension(c *C) {
-	// Test with unknown file extension
-	reporter := &MockResultReporter{}
-	unknownFile := filepath.Join(s.tempDir, "package.unknown")
-
-	err := ioutil.WriteFile(unknownFile, []byte("dummy content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{unknownFile}, reporter)
-
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, unknownFile)
-	c.Check(len(reporter.warnings), Equals, 1)
-	c.Check(strings.Contains(reporter.warnings[0], "Unknown file extension"), Equals, true)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesDirectory(c *C) {
-	// Test with directory containing various files
-	reporter := &MockResultReporter{}
-	subDir := filepath.Join(s.tempDir, "packages")
-	err := os.MkdirAll(subDir, 0755)
-	c.Assert(err, IsNil)
-
-	// Create various file types
-	files := map[string]string{
-		"package1.deb":      "deb content",
-		"package2.udeb":     "udeb content",
-		"source.dsc":        "dsc content",
-		"debug.ddeb":        "ddeb content",
-		"build.buildinfo":   "buildinfo content",
-		"readme.txt":        "text content",
-		"subdir/nested.deb": "nested deb",
-	}
-
-	// Create nested subdirectory
-	nestedDir := filepath.Join(subDir, "subdir")
-	err = os.MkdirAll(nestedDir, 0755)
-	c.Assert(err, IsNil)
-
-	for filename, content := range files {
-		fullPath := filepath.Join(subDir, filename)
-		err := os.MkdirAll(filepath.Dir(fullPath), 0755)
-		c.Assert(err, IsNil)
-		err = ioutil.WriteFile(fullPath, []byte(content), 0644)
-		c.Assert(err, IsNil)
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{subDir}, reporter)
-
-	// Should find package files (sorted)
-	expectedPackageFiles := []string{
-		filepath.Join(subDir, "debug.ddeb"),
-		filepath.Join(subDir, "package1.deb"),
-		filepath.Join(subDir, "package2.udeb"),
-		filepath.Join(subDir, "source.dsc"),
-		filepath.Join(subDir, "subdir", "nested.deb"),
-	}
-	sort.Strings(expectedPackageFiles)
-
-	c.Check(len(packageFiles), Equals, 5)
-	c.Check(packageFiles, DeepEquals, expectedPackageFiles)
-
-	// Should find other files
-	c.Check(len(otherFiles), Equals, 1)
-	c.Check(otherFiles[0], Equals, filepath.Join(subDir, "build.buildinfo"))
-
-	// No failed files
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesMixedLocations(c *C) {
-	// Test with mix of files and directories
-	reporter := &MockResultReporter{}
-
-	// Create individual file
-	debFile := filepath.Join(s.tempDir, "single.deb")
-	err := ioutil.WriteFile(debFile, []byte("single deb"), 0644)
-	c.Assert(err, IsNil)
-
-	// Create directory with files
-	subDir := filepath.Join(s.tempDir, "multi")
-	err = os.MkdirAll(subDir, 0755)
-	c.Assert(err, IsNil)
-
-	dscFile := filepath.Join(subDir, "source.dsc")
-	err = ioutil.WriteFile(dscFile, []byte("dsc content"), 0644)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{debFile, subDir}, reporter)
-
-	expectedFiles := []string{debFile, dscFile}
-	sort.Strings(expectedFiles)
-
-	c.Check(len(packageFiles), Equals, 2)
-	c.Check(packageFiles, DeepEquals, expectedFiles)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesConcurrency(c *C) {
-	// Test concurrent access during directory walking
-	reporter := &MockResultReporter{}
-	subDir := filepath.Join(s.tempDir, "concurrent")
-	err := os.MkdirAll(subDir, 0755)
-	c.Assert(err, IsNil)
-
-	// Create many files to test concurrent access
-	for i := 0; i < 100; i++ {
-		filename := filepath.Join(subDir, fmt.Sprintf("package%d.deb", i))
-		err := ioutil.WriteFile(filename, []byte(fmt.Sprintf("content %d", i)), 0644)
-		c.Assert(err, IsNil)
-
-		if i%10 == 0 {
-			buildinfoFile := filepath.Join(subDir, fmt.Sprintf("build%d.buildinfo", i))
-			err = ioutil.WriteFile(buildinfoFile, []byte(fmt.Sprintf("buildinfo %d", i)), 0644)
-			c.Assert(err, IsNil)
-		}
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{subDir}, reporter)
-
-	c.Check(len(packageFiles), Equals, 100)
-	c.Check(len(otherFiles), Equals, 10) // Every 10th file is buildinfo
-	c.Check(len(failedFiles), Equals, 0)
-
-	// Check that files are sorted
-	c.Check(sort.StringsAreSorted(packageFiles), Equals, true)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesPermissionDenied(c *C) {
-	// Test handling of permission denied errors
-	reporter := &MockResultReporter{}
-
-	// Create directory and remove read permission (if running as non-root)
-	subDir := filepath.Join(s.tempDir, "noperm")
-	err := os.MkdirAll(subDir, 0755)
-	c.Assert(err, IsNil)
-
-	// Create a file inside
-	testFile := filepath.Join(subDir, "test.deb")
-	err = ioutil.WriteFile(testFile, []byte("test"), 0644)
-	c.Assert(err, IsNil)
-
-	// Remove read permission from directory
-	err = os.Chmod(subDir, 0000)
-	if err != nil {
-		c.Skip("Cannot remove permissions, likely running as root")
-	}
-	defer os.Chmod(subDir, 0755) // Restore for cleanup
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{subDir}, reporter)
-
-	// Should handle permission error gracefully
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, subDir)
-	c.Check(len(reporter.warnings), Equals, 1)
-	c.Check(strings.Contains(reporter.warnings[0], "Unable to process"), Equals, true)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesEmptyDirectory(c *C) {
-	// Test with empty directory
-	reporter := &MockResultReporter{}
-	emptyDir := filepath.Join(s.tempDir, "empty")
-	err := os.MkdirAll(emptyDir, 0755)
-	c.Assert(err, IsNil)
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{emptyDir}, reporter)
-
-	c.Check(len(packageFiles), Equals, 0)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesNestedDirectories(c *C) {
-	// Test deeply nested directory structure
-	reporter := &MockResultReporter{}
-
-	// Create nested structure: base/level1/level2/level3/
-	deepDir := filepath.Join(s.tempDir, "base", "level1", "level2", "level3")
-	err := os.MkdirAll(deepDir, 0755)
-	c.Assert(err, IsNil)
-
-	// Place files at different levels
-	files := map[string]string{
-		filepath.Join(s.tempDir, "base", "root.deb"):                               "root",
-		filepath.Join(s.tempDir, "base", "level1", "level1.deb"):                   "level1",
-		filepath.Join(s.tempDir, "base", "level1", "level2", "level2.deb"):         "level2",
-		filepath.Join(s.tempDir, "base", "level1", "level2", "level3", "deep.deb"): "deep",
-	}
-
-	for path, content := range files {
-		err := ioutil.WriteFile(path, []byte(content), 0644)
-		c.Assert(err, IsNil)
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{filepath.Join(s.tempDir, "base")}, reporter)
-
-	c.Check(len(packageFiles), Equals, 4)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-
-	// Verify all nested files were found
-	for expectedPath := range files {
-		found := false
-		for _, foundPath := range packageFiles {
-			if foundPath == expectedPath {
-				found = true
-				break
-			}
-		}
-		c.Check(found, Equals, true, Commentf("File not found: %s", expectedPath))
-	}
-}
-
-func (s *ImportSuite) TestCollectPackageFilesCaseInsensitive(c *C) {
-	// Test case sensitivity of file extensions
-	reporter := &MockResultReporter{}
-
-	// Create files with various case extensions
-	files := []string{
-		"package.deb",
-		"package.DEB",
-		"package.Deb",
-		"source.dsc",
-		"source.DSC",
-		"package.udeb",
-		"package.UDEB",
-		"debug.ddeb",
-		"debug.DDEB",
-	}
-
-	for _, filename := range files {
-		path := filepath.Join(s.tempDir, filename)
-		err := ioutil.WriteFile(path, []byte("content"), 0644)
-		c.Assert(err, IsNil)
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{s.tempDir}, reporter)
-
-	// Only lowercase extensions should be recognized
-	c.Check(len(packageFiles), Equals, 4) // .deb, .dsc, .udeb, .ddeb (lowercase only)
-	c.Check(len(otherFiles), Equals, 0)
-	// Uppercase extensions are silently ignored by the file walker, not reported as failed
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSymlinks(c *C) {
-	// Test handling of symbolic links
-	reporter := &MockResultReporter{}
-
-	// Create a real file
-	realFile := filepath.Join(s.tempDir, "real.deb")
-	err := ioutil.WriteFile(realFile, []byte("real content"), 0644)
-	c.Assert(err, IsNil)
-
-	// Create a symlink to it
-	linkFile := filepath.Join(s.tempDir, "link.deb")
-	err = os.Symlink(realFile, linkFile)
-	if err != nil {
-		c.Skip("Cannot create symlinks on this filesystem")
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{s.tempDir}, reporter)
-
-	// Both real file and symlink should be found
-	c.Check(len(packageFiles), Equals, 2)
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-}
-
-func (s *ImportSuite) TestCollectPackageFilesSpecialCharacters(c *C) {
-	// Test files with special characters in names
-	reporter := &MockResultReporter{}
-
-	// Create files with various special characters
-	specialFiles := []string{
-		"package with spaces.deb",
-		"package-with-dashes.deb",
-		"package_with_underscores.deb",
-		"package.1.0-1.deb",
-		"package+plus.deb",
-		"package@at.deb",
-	}
-
-	for _, filename := range specialFiles {
-		path := filepath.Join(s.tempDir, filename)
-		err := ioutil.WriteFile(path, []byte("content"), 0644)
-		c.Assert(err, IsNil)
-	}
-
-	packageFiles, otherFiles, failedFiles := CollectPackageFiles([]string{s.tempDir}, reporter)
-
-	c.Check(len(packageFiles), Equals, len(specialFiles))
-	c.Check(len(otherFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-}
-
-// Mock implementations for ImportPackageFiles testing
-
-type MockPackagePool struct {
-	importFunc func(string, string, *utils.ChecksumInfo, bool, aptly.ChecksumStorage) (string, error)
-	verifyFunc func(string, string, *utils.ChecksumInfo, aptly.ChecksumStorage) (string, bool, error)
-}
-
-func (m *MockPackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage aptly.ChecksumStorage) (string, error) {
-	if m.importFunc != nil {
-		return m.importFunc(srcPath, basename, checksums, move, storage)
-	}
-	return "pool/" + basename, nil
-}
-
-func (m *MockPackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, storage aptly.ChecksumStorage) (string, bool, error) {
-	if m.verifyFunc != nil {
-		return m.verifyFunc(poolPath, basename, checksums, storage)
-	}
-	return poolPath, true, nil
-}
-
-func (m *MockPackagePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) {
-	return "legacy/" + filename, nil
-}
-
-func (m *MockPackagePool) Size(path string) (int64, error) {
-	return 1024, nil
-}
-
-func (m *MockPackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
-	return nil, nil
-}
-
-func (m *MockPackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
-	return []string{}, nil
-}
-
-func (m *MockPackagePool) Remove(path string) (int64, error) {
-	return 1024, nil
-}
-
-type MockVerifier struct {
-	verifyFunc func(string, string, string) (bool, error)
-}
-
-func (m *MockVerifier) ExtractClearsign(signedMessage string) (string, error) {
-	return signedMessage, nil
-}
-
-func (m *MockVerifier) VerifyClearsign(clearsignInput string, keyringName string, showKeyInfo bool) (string, string, error) {
-	if m.verifyFunc != nil {
-		if valid, err := m.verifyFunc(clearsignInput, keyringName, ""); err != nil {
-			return "", "", err
-		} else if !valid {
-			return "", "", fmt.Errorf("verification failed")
-		}
-	}
-	return clearsignInput, "", nil
-}
-
-// Add missing methods to implement pgp.Verifier interface
-func (m *MockVerifier) InitKeyring(verbose bool) error {
-	return nil
-}
-
-func (m *MockVerifier) AddKeyring(keyring string) {
-	// Mock implementation
-}
-
-func (m *MockVerifier) VerifyDetachedSignature(signature, cleartext io.Reader, showKeyTip bool) error {
-	return nil
-}
-
-func (m *MockVerifier) IsClearSigned(clearsigned io.Reader) (bool, error) {
-	return true, nil
-}
-
-func (m *MockVerifier) VerifyClearsigned(clearsigned io.Reader, showKeyTip bool) (*pgp.KeyInfo, error) {
-	return &pgp.KeyInfo{}, nil
-}
-
-func (m *MockVerifier) ExtractClearsigned(clearsigned io.Reader) (*os.File, error) {
-	// Create a temporary file for mock
-	tmpFile, err := ioutil.TempFile("", "mock_extract")
-	return tmpFile, err
-}
-
-type MockPackageCollection struct {
-	updateFunc func(*Package) error
-	packages   map[string]*Package
-}
-
-func (m *MockPackageCollection) Update(p *Package) error {
-	if m.updateFunc != nil {
-		return m.updateFunc(p)
-	}
-	if m.packages == nil {
-		m.packages = make(map[string]*Package)
-	}
-	m.packages[string(p.Key(""))] = p
-	return nil
-}
-
-func (m *MockPackageCollection) ByKey(key []byte) (*Package, error) {
-	if m.packages == nil {
-		return nil, fmt.Errorf("not found")
-	}
-	if pkg, exists := m.packages[string(key)]; exists {
-		return pkg, nil
-	}
-	return nil, fmt.Errorf("not found")
-}
-
-type MockChecksumStorage struct {
-	getFunc    func(string) (*utils.ChecksumInfo, error)
-	updateFunc func(string, *utils.ChecksumInfo) error
-}
-
-func (m *MockChecksumStorage) Get(path string) (*utils.ChecksumInfo, error) {
-	if m.getFunc != nil {
-		return m.getFunc(path)
-	}
-	return &utils.ChecksumInfo{}, nil
-}
-
-func (m *MockChecksumStorage) Update(path string, c *utils.ChecksumInfo) error {
-	if m.updateFunc != nil {
-		return m.updateFunc(path, c)
-	}
-	return nil
-}
-
-func (s *ImportSuite) TestImportPackageFilesEmptyList(c *C) {
-	// Test ImportPackageFiles with empty file list
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{}, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 0)
-	c.Check(len(reporter.warnings), Equals, 0)
-	c.Check(len(reporter.added), Equals, 0)
-}
-
-func (s *ImportSuite) TestImportPackageFilesNonExistentFile(c *C) {
-	// Test ImportPackageFiles with non-existent file
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	nonExistentFile := filepath.Join(s.tempDir, "nonexistent.deb")
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{nonExistentFile}, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, nonExistentFile)
-	c.Check(len(reporter.warnings), Equals, 1)
-	c.Check(strings.Contains(reporter.warnings[0], "Unable to read file"), Equals, true)
-}
-
-func (s *ImportSuite) TestImportPackageFilesInvalidPackageFile(c *C) {
-	// Test ImportPackageFiles with invalid package file
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Create invalid .deb file
-	invalidDeb := filepath.Join(s.tempDir, "invalid.deb")
-	err := ioutil.WriteFile(invalidDeb, []byte("not a valid deb file"), 0644)
-	c.Assert(err, IsNil)
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{invalidDeb}, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, invalidDeb)
-	c.Check(len(reporter.warnings), Equals, 1)
-	c.Check(strings.Contains(reporter.warnings[0], "Unable to read file"), Equals, true)
-}
-
-func (s *ImportSuite) TestImportPackageFilesPoolImportError(c *C) {
-	// Test ImportPackageFiles with pool import error
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-
-	// Mock pool that fails to import
-	pool := &MockPackagePool{
-		importFunc: func(string, string, *utils.ChecksumInfo, bool, aptly.ChecksumStorage) (string, error) {
-			return "", fmt.Errorf("pool import error")
-		},
-	}
-
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Create a simple .deb file
-	debFile := filepath.Join(s.tempDir, "test.deb")
-	err := ioutil.WriteFile(debFile, []byte("simple deb"), 0644)
-	c.Assert(err, IsNil)
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{debFile}, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(failedFiles[0], Equals, debFile)
-	c.Check(len(reporter.warnings), Equals, 1) // One warning for file processing issue
-}
-
-func (s *ImportSuite) TestImportPackageFilesCollectionUpdateError(c *C) {
-	// Test ImportPackageFiles with collection update error
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-
-	// Use real collection for testing
-	collection := NewPackageCollection(nil)
-
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Create a simple .deb file
-	debFile := filepath.Join(s.tempDir, "test.deb")
-	err := ioutil.WriteFile(debFile, []byte("simple deb"), 0644)
-	c.Assert(err, IsNil)
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{debFile}, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1)
-	c.Check(len(reporter.warnings), Equals, 1) // One warning for file processing issue
-}
-
-func (s *ImportSuite) TestImportPackageFilesForceReplace(c *C) {
-	// Test ImportPackageFiles with force replace option
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Test that forceReplace calls PrepareIndex on the list
-	debFile := filepath.Join(s.tempDir, "test.deb")
-	err := ioutil.WriteFile(debFile, []byte("simple deb"), 0644)
-	c.Assert(err, IsNil)
-
-	// With forceReplace = true
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{debFile}, true, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0) // No files should be processed due to invalid file
-	// Even though the file is invalid, the function should handle forceReplace logic
-	c.Check(len(failedFiles), Equals, 1) // Will fail due to invalid deb file
-}
-
-func (s *ImportSuite) TestImportPackageFilesErrorHandling(c *C) {
-	// Test various error conditions in ImportPackageFiles
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Test with multiple files, some valid some invalid
-	validDeb := filepath.Join(s.tempDir, "valid.deb")
-	invalidDeb := filepath.Join(s.tempDir, "invalid.deb")
-	nonExistent := filepath.Join(s.tempDir, "nonexistent.deb")
-
-	err := ioutil.WriteFile(validDeb, []byte("valid deb content"), 0644)
-	c.Assert(err, IsNil)
-
-	err = ioutil.WriteFile(invalidDeb, []byte("invalid content"), 0644)
-	c.Assert(err, IsNil)
-
-	files := []string{validDeb, invalidDeb, nonExistent}
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, files, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)    // No files should be processed successfully
-	c.Check(len(failedFiles), Equals, 3)       // All files should fail
-	c.Check(len(reporter.warnings), Equals, 3) // Should have warnings for all failures
-}
-
-func (s *ImportSuite) TestImportPackageFilesRestrictionFilter(c *C) {
-	// Test ImportPackageFiles with package restriction filter
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Create mock restriction that rejects all packages
-	restriction := &MockPackageQuery{
-		matchesFunc: func(*Package) bool {
-			return false // Reject all packages
-		},
-	}
-
-	debFile := filepath.Join(s.tempDir, "test.deb")
-	err := ioutil.WriteFile(debFile, []byte("test deb"), 0644)
-	c.Assert(err, IsNil)
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, []string{debFile}, false, verifier, pool, collection, reporter, restriction, checksumProvider)
-
-	c.Check(err, IsNil)
-	c.Check(len(processedFiles), Equals, 0)
-	c.Check(len(failedFiles), Equals, 1) // Should fail due to restriction + invalid file
-	c.Check(len(reporter.warnings) >= 1, Equals, true)
-}
-
-type MockPackageQuery struct {
-	matchesFunc func(*Package) bool
-}
-
-func (m *MockPackageQuery) Matches(p PackageLike) bool {
-	if m.matchesFunc != nil {
-		if pkg, ok := p.(*Package); ok {
-			return m.matchesFunc(pkg)
-		}
-		return false
-	}
-	return true
-}
-
-func (m *MockPackageQuery) Fast(_ PackageCatalog) bool {
-	return false // Mock implementation returns false for simplicity
-}
-
-func (m *MockPackageQuery) Query(list PackageCatalog) *PackageList {
-	return list.Scan(m) // Default implementation
-}
-
-func (m *MockPackageQuery) String() string {
-	return "MockPackageQuery"
-}
-
-func (s *ImportSuite) TestImportPackageFilesFileTypes(c *C) {
-	// Test ImportPackageFiles with different file types
-	list := NewPackageList()
-	reporter := &MockResultReporter{}
-	pool := &MockPackagePool{}
-	collection := NewPackageCollection(nil)
-	verifier := &MockVerifier{}
-	checksumProvider := func(database.ReaderWriter) aptly.ChecksumStorage {
-		return &MockChecksumStorage{}
-	}
-
-	// Create files of different types
-	files := map[string]string{
-		"package.deb":  "deb content",
-		"package.udeb": "udeb content",
-		"source.dsc":   "dsc content",
-		"debug.ddeb":   "ddeb content",
-	}
-
-	var fileList []string
-	for filename, content := range files {
-		path := filepath.Join(s.tempDir, filename)
-		err := ioutil.WriteFile(path, []byte(content), 0644)
-		c.Assert(err, IsNil)
-		fileList = append(fileList, path)
-	}
-
-	processedFiles, failedFiles, err := ImportPackageFiles(
-		list, fileList, false, verifier, pool, collection, reporter, nil, checksumProvider)
-
-	c.Check(err, IsNil)
-	// All files should fail due to invalid format, but function should handle different types
-	c.Check(len(failedFiles), Equals, len(fileList))
-	c.Check(len(processedFiles), Equals, 0)
-}

deb/index_files.go

@@ -253,7 +253,7 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb bool, install
 	if arch == ArchitectureSource {
 		udeb = false
 	}
-	key := fmt.Sprintf("pi-%s-%s-%v-%v-%s", component, arch, udeb, installer, distribution)
+	key := fmt.Sprintf("pi-%s-%s-%v-%v", component, arch, udeb, installer)
 	file, ok := files.indexes[key]
 	if !ok {
 		var relativePath string

deb/index_files_test.go

@@ -1,741 +0,0 @@
-package deb
-
-import (
-	"fmt"
-	"io/ioutil"
-	"strings"
-
-	"github.com/aptly-dev/aptly/aptly"
-	"github.com/aptly-dev/aptly/utils"
-	. "gopkg.in/check.v1"
-)
-
-type IndexFilesSuite struct {
-	tempDir          string
-	publishedStorage *MockPublishedStorage
-	indexFiles       *indexFiles
-}
-
-var _ = Suite(&IndexFilesSuite{})
-
-func (s *IndexFilesSuite) SetUpTest(c *C) {
-	s.tempDir = c.MkDir()
-	s.publishedStorage = &MockPublishedStorage{
-		files:    make(map[string]string),
-		dirs:     make(map[string]bool),
-		links:    make(map[string]string),
-		symlinks: make(map[string]string),
-	}
-	s.indexFiles = newIndexFiles(s.publishedStorage, "dists/test", s.tempDir, "", false, false)
-}
-
-func (s *IndexFilesSuite) TestNewIndexFiles(c *C) {
-	// Test creation of indexFiles struct
-	basePath := "dists/testing"
-	tempDir := "/tmp/test"
-	suffix := ".new"
-	acquireByHash := true
-	skipBz2 := true
-
-	files := newIndexFiles(s.publishedStorage, basePath, tempDir, suffix, acquireByHash, skipBz2)
-
-	c.Check(files.publishedStorage, Equals, s.publishedStorage)
-	c.Check(files.basePath, Equals, basePath)
-	c.Check(files.tempDir, Equals, tempDir)
-	c.Check(files.suffix, Equals, suffix)
-	c.Check(files.acquireByHash, Equals, acquireByHash)
-	c.Check(files.skipBz2, Equals, skipBz2)
-	c.Check(files.renameMap, NotNil)
-	c.Check(files.generatedFiles, NotNil)
-	c.Check(files.indexes, NotNil)
-	c.Check(len(files.renameMap), Equals, 0)
-	c.Check(len(files.generatedFiles), Equals, 0)
-	c.Check(len(files.indexes), Equals, 0)
-}
-
-func (s *IndexFilesSuite) TestIndexFileBufWriter(c *C) {
-	// Test indexFile BufWriter creation
-	file := &indexFile{
-		parent:       s.indexFiles,
-		relativePath: "main/binary-amd64/Packages",
-	}
-
-	// First call should create the writer
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	c.Check(writer, NotNil)
-	c.Check(file.w, Equals, writer)
-	c.Check(file.tempFile, NotNil)
-	c.Check(file.tempFilename, Matches, ".*main_binary-amd64_Packages")
-
-	// Second call should return the same writer
-	writer2, err := file.BufWriter()
-	c.Check(err, IsNil)
-	c.Check(writer2, Equals, writer)
-
-	// Clean up
-	file.tempFile.Close()
-}
-
-func (s *IndexFilesSuite) TestIndexFileBufWriterError(c *C) {
-	// Test BufWriter creation with invalid temp directory
-	invalidFiles := newIndexFiles(s.publishedStorage, "dists/test", "/invalid/path", "", false, false)
-	file := &indexFile{
-		parent:       invalidFiles,
-		relativePath: "main/binary-amd64/Packages",
-	}
-
-	_, err := file.BufWriter()
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Matches, ".*unable to create temporary index file.*")
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalize(c *C) {
-	// Test basic finalization of index file
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/binary-amd64/Packages",
-		compressable:  false,
-		detachedSign:  false,
-		clearSign:     false,
-		acquireByHash: false,
-	}
-
-	// Write some content to the file
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	writer.WriteString("Package: test-package\nVersion: 1.0\n\n")
-
-	err = file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Check that file was published
-	c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages"], NotNil)
-	c.Check(s.publishedStorage.dirs["dists/test/main/binary-amd64"], Equals, true)
-
-	// Check that checksums were generated
-	c.Check(s.indexFiles.generatedFiles["main/binary-amd64/Packages"], NotNil)
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalizeCompressable(c *C) {
-	// Test finalization with compression
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/binary-amd64/Packages",
-		compressable:  true,
-		detachedSign:  false,
-		clearSign:     false,
-		acquireByHash: false,
-		onlyGzip:      false,
-	}
-
-	// Write content and finalize
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	writer.WriteString("Package: test-package\nVersion: 1.0\n\n")
-
-	err = file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Check that compressed files were published
-	c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages"], NotNil)
-	c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages.gz"], NotNil)
-
-	// With skipBz2 = false, should also have .bz2
-	if !s.indexFiles.skipBz2 {
-		c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages.bz2"], NotNil)
-	}
-
-	// Check checksums for all variants
-	c.Check(s.indexFiles.generatedFiles["main/binary-amd64/Packages"], NotNil)
-	c.Check(s.indexFiles.generatedFiles["main/binary-amd64/Packages.gz"], NotNil)
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalizeOnlyGzip(c *C) {
-	// Test finalization with only gzip compression
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/Contents-amd64",
-		compressable:  true,
-		onlyGzip:      true,
-		detachedSign:  false,
-		clearSign:     false,
-		acquireByHash: false,
-	}
-
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	writer.WriteString("some content data\n")
-
-	err = file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Should only have .gz file, not .bz2
-	c.Check(s.publishedStorage.files["dists/test/main/Contents-amd64.gz"], NotNil)
-	_, hasBz2 := s.publishedStorage.files["dists/test/main/Contents-amd64.bz2"]
-	c.Check(hasBz2, Equals, false)
-
-	// Checksums should include both uncompressed and compressed
-	c.Check(s.indexFiles.generatedFiles["main/Contents-amd64"], NotNil)
-	c.Check(s.indexFiles.generatedFiles["main/Contents-amd64.gz"], NotNil)
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalizeDiscardable(c *C) {
-	// Test finalization of discardable file (should create empty file)
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/debian-installer/binary-amd64/Release",
-		discardable:   true,
-		compressable:  false,
-		detachedSign:  false,
-		clearSign:     false,
-		acquireByHash: false,
-	}
-
-	// Don't write any content, just finalize
-	err := file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Should still create the file
-	c.Check(s.publishedStorage.files["dists/test/main/debian-installer/binary-amd64/Release"], NotNil)
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalizeSigning(c *C) {
-	// Test finalization with signing
-	mockSigner := &MockSigner{}
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "Release",
-		compressable:  false,
-		detachedSign:  true,
-		clearSign:     true,
-		acquireByHash: false,
-	}
-
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	writer.WriteString("Suite: test\nCodename: test\n")
-
-	err = file.Finalize(mockSigner)
-	c.Check(err, IsNil)
-
-	// Check that signed files were created
-	c.Check(s.publishedStorage.files["dists/test/Release"], NotNil)
-	c.Check(s.publishedStorage.files["dists/test/Release.gpg"], NotNil)
-	c.Check(s.publishedStorage.files["dists/test/InRelease"], NotNil)
-
-	// Check that signer methods were called
-	c.Check(mockSigner.DetachedSignCalled, Equals, true)
-	c.Check(mockSigner.ClearSignCalled, Equals, true)
-}
-
-func (s *IndexFilesSuite) TestIndexFileFinalizeWithSuffix(c *C) {
-	// Test finalization with suffix (for atomic updates)
-	s.indexFiles.suffix = ".new"
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/binary-amd64/Packages",
-		compressable:  false,
-		detachedSign:  false,
-		clearSign:     false,
-		acquireByHash: false,
-	}
-
-	writer, err := file.BufWriter()
-	c.Check(err, IsNil)
-	writer.WriteString("Package: test\n")
-
-	err = file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Check that file was published with suffix
-	c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages.new"], NotNil)
-
-	// Check that rename mapping was created
-	expectedTarget := "dists/test/main/binary-amd64/Packages"
-	c.Check(s.indexFiles.renameMap["dists/test/main/binary-amd64/Packages.new"], Equals, expectedTarget)
-}
-
-func (s *IndexFilesSuite) TestPackageIndex(c *C) {
-	// Test PackageIndex creation for binary packages
-	file := s.indexFiles.PackageIndex("main", "amd64", false, false, "")
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/binary-amd64/Packages")
-	c.Check(file.compressable, Equals, true)
-	c.Check(file.discardable, Equals, false)
-	c.Check(file.detachedSign, Equals, false)
-
-	// Test that same call returns cached instance
-	file2 := s.indexFiles.PackageIndex("main", "amd64", false, false, "")
-	c.Check(file2, Equals, file)
-}
-
-func (s *IndexFilesSuite) TestPackageIndexSource(c *C) {
-	// Test PackageIndex creation for source packages
-	file := s.indexFiles.PackageIndex("main", ArchitectureSource, false, false, "")
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/source/Sources")
-	c.Check(file.compressable, Equals, true)
-	c.Check(file.discardable, Equals, false)
-}
-
-func (s *IndexFilesSuite) TestPackageIndexUdeb(c *C) {
-	// Test PackageIndex creation for udeb packages
-	file := s.indexFiles.PackageIndex("main", "amd64", true, false, "")
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/debian-installer/binary-amd64/Packages")
-	c.Check(file.compressable, Equals, true)
-}
-
-func (s *IndexFilesSuite) TestPackageIndexInstaller(c *C) {
-	// Test PackageIndex creation for installer images
-	file := s.indexFiles.PackageIndex("main", "amd64", false, true, "")
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/installer-amd64/current/images/SHA256SUMS")
-	c.Check(file.compressable, Equals, false)
-	c.Check(file.detachedSign, Equals, true)
-
-	// Test focal distribution special case
-	fileFocal := s.indexFiles.PackageIndex("main", "amd64", false, true, aptly.DistributionFocal)
-	c.Check(fileFocal, NotNil)
-	c.Check(fileFocal.relativePath, Equals, "main/installer-amd64/current/legacy-images/SHA256SUMS")
-}
-
-func (s *IndexFilesSuite) TestReleaseIndex(c *C) {
-	// Test ReleaseIndex creation for binary architecture
-	file := s.indexFiles.ReleaseIndex("main", "amd64", false)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/binary-amd64/Release")
-	c.Check(file.compressable, Equals, false)
-	c.Check(file.discardable, Equals, false)
-
-	// Test that same call returns cached instance
-	file2 := s.indexFiles.ReleaseIndex("main", "amd64", false)
-	c.Check(file2, Equals, file)
-}
-
-func (s *IndexFilesSuite) TestReleaseIndexSource(c *C) {
-	// Test ReleaseIndex creation for source architecture
-	file := s.indexFiles.ReleaseIndex("main", ArchitectureSource, false)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/source/Release")
-	c.Check(file.compressable, Equals, false)
-}
-
-func (s *IndexFilesSuite) TestReleaseIndexUdeb(c *C) {
-	// Test ReleaseIndex creation for udeb (should be discardable)
-	file := s.indexFiles.ReleaseIndex("main", "amd64", true)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/debian-installer/binary-amd64/Release")
-	c.Check(file.discardable, Equals, true)
-}
-
-func (s *IndexFilesSuite) TestContentsIndex(c *C) {
-	// Test ContentsIndex creation for regular packages
-	file := s.indexFiles.ContentsIndex("main", "amd64", false)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/Contents-amd64")
-	c.Check(file.compressable, Equals, true)
-	c.Check(file.onlyGzip, Equals, true)
-	c.Check(file.discardable, Equals, true)
-
-	// Test that same call returns cached instance
-	file2 := s.indexFiles.ContentsIndex("main", "amd64", false)
-	c.Check(file2, Equals, file)
-}
-
-func (s *IndexFilesSuite) TestContentsIndexUdeb(c *C) {
-	// Test ContentsIndex creation for udeb packages
-	file := s.indexFiles.ContentsIndex("main", "amd64", true)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/Contents-udeb-amd64")
-	c.Check(file.compressable, Equals, true)
-	c.Check(file.onlyGzip, Equals, true)
-}
-
-func (s *IndexFilesSuite) TestContentsIndexSource(c *C) {
-	// Test ContentsIndex for source architecture (should not have udeb)
-	file := s.indexFiles.ContentsIndex("main", ArchitectureSource, true)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/Contents-source")
-	// udeb flag should be ignored for source
-}
-
-func (s *IndexFilesSuite) TestLegacyContentsIndex(c *C) {
-	// Test LegacyContentsIndex creation
-	file := s.indexFiles.LegacyContentsIndex("amd64", false)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "Contents-amd64")
-	c.Check(file.compressable, Equals, true)
-	c.Check(file.onlyGzip, Equals, true)
-	c.Check(file.discardable, Equals, true)
-
-	// Test that same call returns cached instance
-	file2 := s.indexFiles.LegacyContentsIndex("amd64", false)
-	c.Check(file2, Equals, file)
-}
-
-func (s *IndexFilesSuite) TestLegacyContentsIndexUdeb(c *C) {
-	// Test LegacyContentsIndex for udeb
-	file := s.indexFiles.LegacyContentsIndex("amd64", true)
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "Contents-udeb-amd64")
-}
-
-func (s *IndexFilesSuite) TestSkelIndex(c *C) {
-	// Test SkelIndex creation
-	file := s.indexFiles.SkelIndex("main", "extra/file.txt")
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "main/extra/file.txt")
-	c.Check(file.compressable, Equals, false)
-	c.Check(file.discardable, Equals, false)
-
-	// Test that same call returns cached instance
-	file2 := s.indexFiles.SkelIndex("main", "extra/file.txt")
-	c.Check(file2, Equals, file)
-}
-
-func (s *IndexFilesSuite) TestReleaseFile(c *C) {
-	// Test ReleaseFile creation (should not be cached)
-	file := s.indexFiles.ReleaseFile()
-	c.Check(file, NotNil)
-	c.Check(file.relativePath, Equals, "Release")
-	c.Check(file.compressable, Equals, false)
-	c.Check(file.detachedSign, Equals, true)
-	c.Check(file.clearSign, Equals, true)
-
-	// Test that new call returns different instance (not cached)
-	file2 := s.indexFiles.ReleaseFile()
-	c.Check(file2, Not(Equals), file)
-	c.Check(file2.relativePath, Equals, "Release")
-}
-
-func (s *IndexFilesSuite) TestFinalizeAll(c *C) {
-	// Test finalizing all index files
-	mockSigner := &MockSigner{}
-	mockProgress := &MockProgress{}
-
-	// Create some index files
-	file1 := s.indexFiles.PackageIndex("main", "amd64", false, false, "")
-	file2 := s.indexFiles.ContentsIndex("main", "amd64", false)
-
-	// Write content to files
-	writer1, _ := file1.BufWriter()
-	writer1.WriteString("Package: test1\n")
-	writer2, _ := file2.BufWriter()
-	writer2.WriteString("test1 section/file")
-
-	err := s.indexFiles.FinalizeAll(mockProgress, mockSigner)
-	c.Check(err, IsNil)
-
-	// Check that files were published
-	c.Check(len(s.publishedStorage.files) > 0, Equals, true)
-
-	// Check that progress was tracked
-	c.Check(mockProgress.InitBarCalled, Equals, true)
-	c.Check(mockProgress.ShutdownBarCalled, Equals, true)
-	c.Check(mockProgress.AddBarCount >= 2, Equals, true)
-
-	// Check that indexes map is cleared
-	c.Check(len(s.indexFiles.indexes), Equals, 0)
-}
-
-func (s *IndexFilesSuite) TestFinalizeAllNoProgress(c *C) {
-	// Test finalizing without progress tracking
-	file := s.indexFiles.PackageIndex("main", "amd64", false, false, "")
-	writer, _ := file.BufWriter()
-	writer.WriteString("Package: test\n")
-
-	err := s.indexFiles.FinalizeAll(nil, nil)
-	c.Check(err, IsNil)
-
-	c.Check(len(s.publishedStorage.files) > 0, Equals, true)
-	c.Check(len(s.indexFiles.indexes), Equals, 0)
-}
-
-func (s *IndexFilesSuite) TestRenameFiles(c *C) {
-	// Test file renaming functionality
-	s.indexFiles.renameMap["old/path"] = "new/path"
-	s.indexFiles.renameMap["another/old"] = "another/new"
-
-	err := s.indexFiles.RenameFiles()
-	c.Check(err, IsNil)
-
-	// Check that rename operations were performed
-	c.Check(s.publishedStorage.RenameOperations["old/path"], Equals, "new/path")
-	c.Check(s.publishedStorage.RenameOperations["another/old"], Equals, "another/new")
-}
-
-func (s *IndexFilesSuite) TestRenameFilesError(c *C) {
-	// Test rename error handling
-	s.publishedStorage.SimulateRenameError = true
-	s.indexFiles.renameMap["will/fail"] = "target"
-
-	err := s.indexFiles.RenameFiles()
-	c.Check(err, NotNil)
-	c.Check(err.Error(), Matches, ".*unable to rename.*")
-}
-
-func (s *IndexFilesSuite) TestAcquireByHashFeature(c *C) {
-	// Test acquire-by-hash functionality
-	s.indexFiles.acquireByHash = true
-
-	file := &indexFile{
-		parent:        s.indexFiles,
-		relativePath:  "main/binary-amd64/Packages",
-		compressable:  true,
-		acquireByHash: true,
-	}
-
-	writer, _ := file.BufWriter()
-	writer.WriteString("Package: test-hash\nVersion: 1.0\n")
-
-	err := file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Check that by-hash directories were created
-	c.Check(s.publishedStorage.dirs["dists/test/main/binary-amd64/by-hash/MD5Sum"], Equals, true)
-	c.Check(s.publishedStorage.dirs["dists/test/main/binary-amd64/by-hash/SHA1"], Equals, true)
-	c.Check(s.publishedStorage.dirs["dists/test/main/binary-amd64/by-hash/SHA256"], Equals, true)
-	c.Check(s.publishedStorage.dirs["dists/test/main/binary-amd64/by-hash/SHA512"], Equals, true)
-}
-
-func (s *IndexFilesSuite) TestPackageIndexByHashFunction(c *C) {
-	// Test packageIndexByHash function directly
-	s.indexFiles.generatedFiles["main/binary-amd64/Packages"] = utils.ChecksumInfo{
-		MD5:    "d41d8cd98f00b204e9800998ecf8427e",
-		SHA1:   "da39a3ee5e6b4b0d3255bfef95601890afd80709",
-		SHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-		SHA512: "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
-	}
-
-	file := &indexFile{
-		parent:       s.indexFiles,
-		relativePath: "main/binary-amd64/Packages",
-	}
-
-	err := packageIndexByHash(file, "", "SHA256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-	c.Check(err, IsNil)
-
-	// Check that hard link was created
-	expectedSrc := "dists/test/main/binary-amd64/Packages"
-	expectedDst := "dists/test/main/binary-amd64/by-hash/SHA256/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-	c.Check(s.publishedStorage.HardLinks[expectedDst], Equals, expectedSrc)
-}
-
-func (s *IndexFilesSuite) TestSkipBz2Feature(c *C) {
-	// Test skipBz2 functionality
-	s.indexFiles.skipBz2 = true
-
-	file := &indexFile{
-		parent:       s.indexFiles,
-		relativePath: "main/binary-amd64/Packages",
-		compressable: true,
-		onlyGzip:     false,
-	}
-
-	writer, _ := file.BufWriter()
-	writer.WriteString("Package: no-bz2\n")
-
-	err := file.Finalize(nil)
-	c.Check(err, IsNil)
-
-	// Should have .gz but not .bz2
-	c.Check(s.publishedStorage.files["dists/test/main/binary-amd64/Packages.gz"], NotNil)
-	_, hasBz2 := s.publishedStorage.files["dists/test/main/binary-amd64/Packages.bz2"]
-	c.Check(hasBz2, Equals, false)
-}
-
-// Mock implementations for testing
-
-type MockPublishedStorage struct {
-	files                 map[string]string
-	dirs                  map[string]bool
-	links                 map[string]string
-	symlinks              map[string]string
-	HardLinks             map[string]string
-	RenameOperations      map[string]string
-	SimulateRenameError   bool
-	SimulateFileError     bool
-	SimulateSymlinkExists bool
-}
-
-func (m *MockPublishedStorage) MkDir(path string) error {
-	if m.dirs == nil {
-		m.dirs = make(map[string]bool)
-	}
-	m.dirs[path] = true
-	return nil
-}
-
-func (m *MockPublishedStorage) PutFile(path, source string) error {
-	if m.SimulateFileError {
-		return fmt.Errorf("simulated file error")
-	}
-	if m.files == nil {
-		m.files = make(map[string]string)
-	}
-	// Read source content (simplified for test)
-	content, err := ioutil.ReadFile(source)
-	if err != nil {
-		// Create dummy content for missing files
-		content = []byte("mock content")
-	}
-	m.files[path] = string(content)
-	return nil
-}
-
-func (m *MockPublishedStorage) Remove(path string) error {
-	delete(m.files, path)
-	delete(m.links, path)
-	delete(m.symlinks, path)
-	return nil
-}
-
-func (m *MockPublishedStorage) RenameFile(oldName, newName string) error {
-	if m.SimulateRenameError {
-		return fmt.Errorf("simulated rename error")
-	}
-	if m.RenameOperations == nil {
-		m.RenameOperations = make(map[string]string)
-	}
-	m.RenameOperations[oldName] = newName
-	if content, exists := m.files[oldName]; exists {
-		m.files[newName] = content
-		delete(m.files, oldName)
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) FileExists(path string) (bool, error) {
-	_, exists := m.files[path]
-	if !exists {
-		_, exists = m.symlinks[path]
-	}
-	if m.SimulateSymlinkExists {
-		return true, nil
-	}
-	return exists, nil
-}
-
-func (m *MockPublishedStorage) HardLink(src, dst string) error {
-	if m.HardLinks == nil {
-		m.HardLinks = make(map[string]string)
-	}
-	m.HardLinks[dst] = src
-	return nil
-}
-
-func (m *MockPublishedStorage) SymLink(src, dst string) error {
-	if m.symlinks == nil {
-		m.symlinks = make(map[string]string)
-	}
-	m.symlinks[dst] = src
-	return nil
-}
-
-func (m *MockPublishedStorage) ReadLink(path string) (string, error) {
-	if target, exists := m.symlinks[path]; exists {
-		return target, nil
-	}
-	return "", fmt.Errorf("not a symlink")
-}
-
-func (m *MockPublishedStorage) Filelist(prefix string) ([]string, error) {
-	var files []string
-	for path := range m.files {
-		if strings.HasPrefix(path, prefix) {
-			files = append(files, path)
-		}
-	}
-	return files, nil
-}
-
-func (m *MockPublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
-	// Mock implementation - just track that it was called
-	if m.files == nil {
-		m.files = make(map[string]string)
-	}
-	m.files[publishedRelPath] = "linked from pool"
-	return nil
-}
-
-func (m *MockPublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
-	// Mock implementation - remove files with path prefix
-	for filePath := range m.files {
-		if strings.HasPrefix(filePath, path) {
-			delete(m.files, filePath)
-		}
-	}
-	for dirPath := range m.dirs {
-		if strings.HasPrefix(dirPath, path) {
-			delete(m.dirs, dirPath)
-		}
-	}
-	return nil
-}
-
-func (m *MockPublishedStorage) Flush() error {
-	return nil
-}
-
-type MockSigner struct {
-	DetachedSignCalled bool
-	ClearSignCalled    bool
-}
-
-func (m *MockSigner) Init() error                                     { return nil }
-func (m *MockSigner) SetKey(keyRef string)                            {}
-func (m *MockSigner) SetKeyRing(keyring, secretKeyring string)        {}
-func (m *MockSigner) SetPassphrase(passphrase, passphraseFile string) {}
-func (m *MockSigner) SetBatch(batch bool)                             {}
-
-func (m *MockSigner) DetachedSign(source, signature string) error {
-	m.DetachedSignCalled = true
-	// Create mock signature file
-	return ioutil.WriteFile(signature, []byte("mock signature"), 0644)
-}
-
-func (m *MockSigner) ClearSign(source, signature string) error {
-	m.ClearSignCalled = true
-	// Create mock clear-signed file
-	return ioutil.WriteFile(signature, []byte("mock clear signature"), 0644)
-}
-
-type MockProgress struct {
-	InitBarCalled     bool
-	ShutdownBarCalled bool
-	AddBarCount       int
-}
-
-func (m *MockProgress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
-	m.InitBarCalled = true
-}
-
-func (m *MockProgress) ShutdownBar() {
-	m.ShutdownBarCalled = true
-}
-
-func (m *MockProgress) AddBar(count int) {
-	m.AddBarCount += count
-}
-
-func (m *MockProgress) SetBar(count int) {}
-
-func (m *MockProgress) PrintfBar(msg string, a ...interface{}) {}
-
-func (m *MockProgress) ColoredPrintf(msg string, a ...interface{}) {}
-
-func (m *MockProgress) Printf(msg string, a ...interface{}) {}
-
-func (m *MockProgress) Flush() {}
-
-func (m *MockProgress) PrintfStdErr(msg string, a ...interface{}) {}
-
-func (m *MockProgress) Start() {}
-
-func (m *MockProgress) Shutdown() {}
-
-func (m *MockProgress) Write(p []byte) (n int, err error) {
-	return len(p), nil
-}

deb/list.go

@@ -598,6 +598,7 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) {
 					//
 					// when follow-all-variants is enabled, we need to try to expand anyway,
 					// as even if dependency is satisfied now, there might be other ways to satisfy dependency
+					// FIXME: do not search twice
 					if result.Search(dep, false, true) != nil {
 						if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
 							options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true))

deb/local.go

@@ -168,6 +168,8 @@ func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
 
 // LoadComplete loads additional information for local repo
 func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
+	repo.packageRefs = &PackageRefList{}
+
 	encoded, err := collection.db.Get(repo.RefKey())
 	if err == database.ErrNotFound {
 		return nil
@@ -176,7 +178,6 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
 		return err
 	}
 
-	repo.packageRefs = &PackageRefList{}
 	return repo.packageRefs.Decode(encoded)
 }
 

deb/local_test.go

@@ -133,6 +133,18 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
 	c.Assert(r.String(), Equals, repo.String())
 }
 
+func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) {
+	repo := NewLocalRepo("local1", "Comment 1")
+	c.Assert(s.collection.Update(repo), IsNil)
+
+	r, err := s.collection.ByName("local1")
+	c.Assert(err, IsNil)
+
+	c.Assert(s.collection.LoadComplete(r), IsNil)
+	c.Assert(r.packageRefs, NotNil)
+	c.Assert(r.NumPackages(), Equals, 0)
+}
+
 func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
 	repo := NewLocalRepo("local1", "Comment 1")
 	c.Assert(s.collection.Update(repo), IsNil)

deb/package_deps_test.go

@@ -1,348 +0,0 @@
-package deb
-
-import (
-	. "gopkg.in/check.v1"
-)
-
-type PackageDependenciesSuite struct{}
-
-var _ = Suite(&PackageDependenciesSuite{})
-
-func (s *PackageDependenciesSuite) TestParseDependenciesBasic(c *C) {
-	// Test basic dependency parsing with single dependency
-	stanza := Stanza{
-		"Depends": "package1",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1"})
-
-	// Check that key was removed from stanza
-	_, exists := stanza["Depends"]
-	c.Check(exists, Equals, false)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesMultiple(c *C) {
-	// Test parsing multiple dependencies separated by commas
-	stanza := Stanza{
-		"Depends": "package1, package2, package3",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1", "package2", "package3"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesWithVersions(c *C) {
-	// Test parsing dependencies with version constraints
-	stanza := Stanza{
-		"Depends": "package1 (>= 1.0), package2 (<< 2.0), package3 (= 1.5)",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1 (>= 1.0)", "package2 (<< 2.0)", "package3 (= 1.5)"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesWithWhitespace(c *C) {
-	// Test parsing dependencies with various whitespace patterns
-	stanza := Stanza{
-		"Depends": "  package1  ,   package2   ,package3,  package4  ",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1", "package2", "package3", "package4"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesEmpty(c *C) {
-	// Test parsing empty dependency string
-	stanza := Stanza{
-		"Depends": "",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, IsNil)
-
-	// Check that key was removed from stanza
-	_, exists := stanza["Depends"]
-	c.Check(exists, Equals, false)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesWhitespaceOnly(c *C) {
-	// Test parsing dependency string with only whitespace
-	stanza := Stanza{
-		"Depends": "   \t  \n  ",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, IsNil)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesMissingKey(c *C) {
-	// Test parsing when key doesn't exist in stanza
-	stanza := Stanza{
-		"SomeOtherField": "value",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, IsNil)
-
-	// Check that original stanza is unchanged
-	_, exists := stanza["SomeOtherField"]
-	c.Check(exists, Equals, true)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesComplexFormat(c *C) {
-	// Test parsing complex dependency formats
-	stanza := Stanza{
-		"Depends": "libc6 (>= 2.17), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.8.1)",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"libc6 (>= 2.17)",
-		"libgcc1 (>= 1:4.1.1)",
-		"libstdc++6 (>= 4.8.1)",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesAlternatives(c *C) {
-	// Test parsing dependencies with alternatives (| separator within single dependency)
-	stanza := Stanza{
-		"Depends": "mail-transport-agent | postfix, libc6 (>= 2.17)",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"mail-transport-agent | postfix",
-		"libc6 (>= 2.17)",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesSpecialCharacters(c *C) {
-	// Test parsing dependencies with special characters in package names
-	stanza := Stanza{
-		"Depends": "lib-package++-dev, package.name, package_underscore",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"lib-package++-dev",
-		"package.name",
-		"package_underscore",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesArchitectures(c *C) {
-	// Test parsing dependencies with architecture specifications
-	stanza := Stanza{
-		"Depends": "package1 [amd64], package2 [!arm64], package3 [i386 amd64]",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"package1 [amd64]",
-		"package2 [!arm64]",
-		"package3 [i386 amd64]",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesProfiles(c *C) {
-	// Test parsing dependencies with build profiles
-	stanza := Stanza{
-		"Depends": "package1 <cross>, package2 <!nocheck>, package3 <stage1 !cross>",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"package1 <cross>",
-		"package2 <!nocheck>",
-		"package3 <stage1 !cross>",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesLongLine(c *C) {
-	// Test parsing very long dependency line
-	longDeps := "pkg1, pkg2, pkg3, pkg4, pkg5, pkg6, pkg7, pkg8, pkg9, pkg10, " +
-		"pkg11, pkg12, pkg13, pkg14, pkg15, pkg16, pkg17, pkg18, pkg19, pkg20"
-
-	stanza := Stanza{
-		"Depends": longDeps,
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(len(result), Equals, 20)
-	c.Check(result[0], Equals, "pkg1")
-	c.Check(result[19], Equals, "pkg20")
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesSingleComma(c *C) {
-	// Test edge case with single comma
-	stanza := Stanza{
-		"Depends": ",",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"", ""})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesTrailingComma(c *C) {
-	// Test with trailing comma
-	stanza := Stanza{
-		"Depends": "package1, package2,",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1", "package2", ""})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesLeadingComma(c *C) {
-	// Test with leading comma
-	stanza := Stanza{
-		"Depends": ", package1, package2",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"", "package1", "package2"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesMultipleCommas(c *C) {
-	// Test with multiple consecutive commas
-	stanza := Stanza{
-		"Depends": "package1,, package2,,, package3",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1", "", "package2", "", "", "package3"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesRealWorld(c *C) {
-	// Test with real-world dependency examples
-	stanza := Stanza{
-		"Depends": "debconf (>= 0.5) | debconf-2.0, libc6 (>= 2.14), libgcc1 (>= 1:3.0), libstdc++6 (>= 5.2)",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"debconf (>= 0.5) | debconf-2.0",
-		"libc6 (>= 2.14)",
-		"libgcc1 (>= 1:3.0)",
-		"libstdc++6 (>= 5.2)",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesDifferentKeys(c *C) {
-	// Test parsing different dependency types
-	stanza := Stanza{
-		"Depends":             "runtime-dep",
-		"Build-Depends":       "build-dep",
-		"Build-Depends-Indep": "build-indep-dep",
-		"Pre-Depends":         "pre-dep",
-		"Suggests":            "suggest-dep",
-		"Recommends":          "recommend-dep",
-	}
-
-	// Test each dependency type
-	depends := parseDependencies(stanza, "Depends")
-	c.Check(depends, DeepEquals, []string{"runtime-dep"})
-
-	buildDepends := parseDependencies(stanza, "Build-Depends")
-	c.Check(buildDepends, DeepEquals, []string{"build-dep"})
-
-	buildDependsIndep := parseDependencies(stanza, "Build-Depends-Indep")
-	c.Check(buildDependsIndep, DeepEquals, []string{"build-indep-dep"})
-
-	preDepends := parseDependencies(stanza, "Pre-Depends")
-	c.Check(preDepends, DeepEquals, []string{"pre-dep"})
-
-	suggests := parseDependencies(stanza, "Suggests")
-	c.Check(suggests, DeepEquals, []string{"suggest-dep"})
-
-	recommends := parseDependencies(stanza, "Recommends")
-	c.Check(recommends, DeepEquals, []string{"recommend-dep"})
-
-	// Verify all keys were removed
-	c.Check(len(stanza), Equals, 0)
-}
-
-func (s *PackageDependenciesSuite) TestPackageDependenciesStruct(c *C) {
-	// Test PackageDependencies struct creation and field access
-	deps := PackageDependencies{
-		Depends:           []string{"dep1", "dep2"},
-		BuildDepends:      []string{"build-dep1", "build-dep2"},
-		BuildDependsInDep: []string{"build-indep-dep1"},
-		PreDepends:        []string{"pre-dep1"},
-		Suggests:          []string{"suggest1", "suggest2"},
-		Recommends:        []string{"recommend1"},
-	}
-
-	c.Check(deps.Depends, DeepEquals, []string{"dep1", "dep2"})
-	c.Check(deps.BuildDepends, DeepEquals, []string{"build-dep1", "build-dep2"})
-	c.Check(deps.BuildDependsInDep, DeepEquals, []string{"build-indep-dep1"})
-	c.Check(deps.PreDepends, DeepEquals, []string{"pre-dep1"})
-	c.Check(deps.Suggests, DeepEquals, []string{"suggest1", "suggest2"})
-	c.Check(deps.Recommends, DeepEquals, []string{"recommend1"})
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesUnicodeCharacters(c *C) {
-	// Test parsing dependencies with unicode characters
-	stanza := Stanza{
-		"Depends": "libμ-package, package-ñoño, 中文-package",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	expected := []string{
-		"libμ-package",
-		"package-ñoño",
-		"中文-package",
-	}
-	c.Check(result, DeepEquals, expected)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesStanzaImmutability(c *C) {
-	// Test that original stanza values are not modified (except for key removal)
-	original := Stanza{
-		"Depends": "package1, package2",
-		"Other":   "value",
-	}
-
-	// Make a copy to compare
-	stanza := Stanza{
-		"Depends": original["Depends"],
-		"Other":   original["Other"],
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, DeepEquals, []string{"package1", "package2"})
-
-	// Check that Depends key was removed but Other remains unchanged
-	_, dependsExists := stanza["Depends"]
-	c.Check(dependsExists, Equals, false)
-	c.Check(stanza["Other"], Equals, original["Other"])
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesEmptyStanza(c *C) {
-	// Test with completely empty stanza
-	stanza := Stanza{}
-
-	result := parseDependencies(stanza, "Depends")
-	c.Check(result, IsNil)
-	c.Check(len(stanza), Equals, 0)
-}
-
-func (s *PackageDependenciesSuite) TestParseDependenciesTabsAndNewlines(c *C) {
-	// Test parsing dependencies with tabs and newlines
-	stanza := Stanza{
-		"Depends": "package1,\n\tpackage2,\t package3\n,package4",
-	}
-
-	result := parseDependencies(stanza, "Depends")
-	// The function should handle tabs and newlines as whitespace
-	c.Check(len(result), Equals, 4)
-	c.Check(result[0], Equals, "package1")
-	c.Check(result[3], Equals, "package4")
-}

deb/ppa.go

@@ -28,6 +28,11 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 		}
 	}
 
+	baseurl := config.PpaBaseURL
+	if baseurl == "" {
+		baseurl = "http://ppa.launchpad.net"
+	}
+
 	codename := config.PpaCodename
 	if codename == "" {
 		codename, err = getCodename()
@@ -39,7 +44,7 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib
 
 	distribution = codename
 	components = []string{"main"}
-	url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID)
+	url = fmt.Sprintf("%s/%s/%s/%s", baseurl, matches[1], matches[2], distributorID)
 
 	return
 }

deb/publish.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -603,6 +604,15 @@ func (p *PublishedRepo) Key() []byte {
 	return []byte("U" + p.StoragePrefix() + ">>" + p.Distribution)
 }
 
+// PrefixPoolLockKey returns the task-queue resource key that serialises all
+// publish operations sharing the same pool directory under storagePrefix.
+// It must be held whenever a non-MultiDist publish may read or clean the
+// shared pool, to prevent concurrent cleanup runs from deleting each other's
+// files.  See docs/Resource-Locking.md for the full key-namespace table.
+func PrefixPoolLockKey(storagePrefix string) string {
+	return "P" + storagePrefix
+}
+
 // RefKey is a unique id for package reference list
 func (p *PublishedRepo) RefKey(component string) []byte {
 	return []byte("E" + p.UUID + component)
@@ -814,9 +824,12 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri
 // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
 func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
 	collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
-	err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
+	err = publishedStorage.MkDir(filepath.Join(p.Prefix, "pool"))
 	if err != nil {
 		return err
 	}
@@ -1126,7 +1139,15 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 	release["Label"] = p.GetLabel()
 	release["Suite"] = p.GetSuite()
 	release["Codename"] = p.GetCodename()
-	release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
+	datetimeFormat := "Mon, 2 Jan 2006 15:04:05 MST"
+
+	publishDate := time.Now().UTC()
+	if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
+		if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil {
+			publishDate = time.Unix(sec, 0).UTC()
+		}
+	}
+	release["Date"] = publishDate.Format(datetimeFormat)
 	release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
 	if p.AcquireByHash {
 		release["Acquire-By-Hash"] = "yes"
@@ -1174,12 +1195,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 		return err
 	}
 
-	// Flush any pending uploads before renaming files
-	err = publishedStorage.Flush()
-	if err != nil {
-		return fmt.Errorf("error flushing pending uploads: %s", err)
-	}
-
 	return indexes.RenameFiles()
 }
 
@@ -1188,7 +1203,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
 // It can remove prefix fully, and part of pool (for specific component)
 func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool,
 	removePoolComponents []string, progress aptly.Progress) error {
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage)
+	if err != nil {
+		return err
+	}
 
 	// I. Easy: remove whole prefix (meta+packages)
 	if removePrefix {
@@ -1201,7 +1219,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor
 	}
 
 	// II. Medium: remove metadata, it can't be shared as prefix/distribution as unique
-	err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
+	err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress)
 	if err != nil {
 		return err
 	}
@@ -1528,6 +1546,55 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
 	return referencedFiles, nil
 }
 
+// CleanupAfterMultiDistToggle cleans up stale pool files left behind when the
+// MultiDist flag is toggled on a published repository.
+//
+//   - false→true: Publish() wrote packages into pool/<distribution>/<component>/
+//     but the old flat pool/<component>/ files were not removed because
+//     CleanupPrefixComponentFiles only scans the new MultiDist tree.
+//     A second pass with MultiDist=false cleans the legacy flat layout by
+//     reusing the existing orphan-detection logic (the repo is now MultiDist=true
+//     so it is excluded from the referenced-files scan, making its old pool
+//     entries appear orphaned).
+//
+//   - true→false: Publish() wrote packages into pool/<component>/ but the old
+//     per-distribution pool/<distribution>/<component>/ directories were not
+//     removed.  The orphan-detection approach cannot be used here because the
+//     repo's RefList still contains all packages (they just moved locations).
+//     Instead we directly remove each pool/<distribution>/<component>/ directory.
+//     This is safe because per-distribution pool dirs are exclusive to a single
+//     prefix+distribution combination — no other published repo can share them.
+func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(publishedStorageProvider aptly.PublishedStorageProvider,
+	published *PublishedRepo, prevMultiDist bool, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
+	if prevMultiDist == published.MultiDist {
+		return nil
+	}
+
+	if !prevMultiDist && published.MultiDist {
+		// false→true: use orphan-detection via the existing cleanup, but with
+		// MultiDist temporarily set to false so it scans the flat pool layout.
+		legacy := *published
+		legacy.MultiDist = false
+		return collection.CleanupPrefixComponentFiles(publishedStorageProvider, &legacy, cleanComponents, collectionFactory, progress)
+	}
+
+	// true→false: directly remove the per-distribution pool directories.
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
+	for _, component := range cleanComponents {
+		poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component)
+		if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil {
+			return err
+		}
+	}
+	// Remove the distribution-level pool dir if it is now empty.
+	distPoolDir := filepath.Join(published.Prefix, "pool", published.Distribution)
+	_ = publishedStorage.RemoveDirs(distPoolDir, progress)
+	return nil
+}
+
 // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
 func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider,
 	published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error {
@@ -1541,7 +1608,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
 	distribution := published.Distribution
 
 	rootPath := filepath.Join(prefix, "dists", distribution)
-	publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage)
+	if err != nil {
+		return err
+	}
 
 	sort.Strings(cleanComponents)
 	publishedComponents := published.Components()

deb/publish_test.go

@@ -61,12 +61,12 @@ type FakeStorageProvider struct {
 	storages map[string]aptly.PublishedStorage
 }
 
-func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage {
+func (p *FakeStorageProvider) GetPublishedStorage(name string) (aptly.PublishedStorage, error) {
 	storage, ok := p.storages[name]
 	if !ok {
-		panic(fmt.Sprintf("unknown storage: %#v", name))
+		return nil, fmt.Errorf("unknown storage: %#v", name)
 	}
-	return storage
+	return storage, nil
 }
 
 type PublishedRepoSuite struct {
@@ -433,6 +433,47 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
 	c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
 }
 
+func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) {
+	// Test with SOURCE_DATE_EPOCH set
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC
+	c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
+func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) {
+	// Test with invalid SOURCE_DATE_EPOCH (should fallback to current time)
+	_ = os.Setenv("SOURCE_DATE_EPOCH", "invalid")
+	defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
+
+	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
+	c.Assert(err, IsNil)
+
+	rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"))
+	c.Assert(err, IsNil)
+	defer func() { _ = rf.Close() }()
+
+	cfr := NewControlFileReader(rf, true, false)
+	st, err := cfr.ReadStanza()
+	c.Assert(err, IsNil)
+
+	// Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH)
+	c.Check(st["Date"], Not(Equals), "")
+	c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC")
+}
+
 func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
 	err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
 	c.Assert(err, IsNil)
@@ -756,7 +797,10 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 	snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
 	_ = s.snapshotCollection.Add(snap3)
 
-	// Ensure that adding a second publish point with matching files doesn't give duplicate results.
+	// When a second publish point references the same package (snap3 is a clone of snap2,
+	// both containing p3/lonely-strangers), listReferencedFilesByComponent deduplicates by
+	// package ref so the file appears only once.  StrSlicesSubstract handles a single entry
+	// correctly, so no duplicate is needed for cleanup safety.
 	repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
 	c.Check(err, IsNil)
 	c.Check(s.collection.Add(repo3), IsNil)
@@ -771,7 +815,9 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
 			"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
 			"a/alien-arena/mars-invaders_7.40-2_i386.deb",
 		},
-		"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
+		"main": {
+			"a/alien-arena/lonely-strangers_7.40-2_i386.deb",
+		},
 	})
 }
 

deb/reflist.go

@@ -79,6 +79,9 @@ func (l *PackageRefList) Decode(input []byte) error {
 
 // ForEach calls handler for each package ref in list
 func (l *PackageRefList) ForEach(handler func([]byte) error) error {
+	if l == nil {
+		return nil
+	}
 	var err error
 	for _, p := range l.Refs {
 		err = handler(p)

deb/reflist_test.go

@@ -130,6 +130,17 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) {
 	c.Check(err, Equals, e)
 }
 
+func (s *PackageRefListSuite) TestForEachNilList(c *C) {
+	var l *PackageRefList
+	called := false
+	err := l.ForEach(func([]byte) error {
+		called = true
+		return nil
+	})
+	c.Assert(err, IsNil)
+	c.Assert(called, Equals, false)
+}
+
 func (s *PackageRefListSuite) TestHas(c *C) {
 	_ = s.list.Add(s.p1)
 	_ = s.list.Add(s.p3)

deb/remote.go

@@ -574,7 +574,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
 					if progress != nil {
 						progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p)
 					}
-				} else if err != nil {
+				} else {
 					return err
 				}
 			}

deb/snapshot.go

@@ -125,12 +125,6 @@ func (s *Snapshot) Key() []byte {
 	return []byte("S" + s.UUID)
 }
 
-// ResourceKey is a unique identifier of the resource
-// this snapshot uses. Instead of uuid it uses name
-// which needs to be unique as well.
-func (s *Snapshot) ResourceKey() []byte {
-	return []byte("S" + s.Name)
-}
 
 // RefKey is a unique id for package reference list
 func (s *Snapshot) RefKey() []byte {

deb/version.go

@@ -30,16 +30,16 @@ func CompareVersions(ver1, ver2 string) int {
 
 // parseVersions breaks down full version to components (possibly empty)
 func parseVersion(ver string) (epoch, upstream, debian string) {
-	i := strings.LastIndex(ver, "-")
-	if i != -1 {
-		debian, ver = ver[i+1:], ver[:i]
-	}
-
-	i = strings.Index(ver, ":")
+	i := strings.Index(ver, ":")
 	if i != -1 {
 		epoch, ver = ver[:i], ver[i+1:]
 	}
 
+	i = strings.Index(ver, "-")
+	if i != -1 {
+		debian, ver = ver[i+1:], ver[:i]
+	}
+
 	upstream = ver
 
 	return

deb/version_test.go

@@ -20,10 +20,10 @@ func (s *VersionSuite) TestParseVersion(c *C) {
 	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3.4", "1"})
 
 	e, u, d = parseVersion("1.3-pre4-1")
-	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3-pre4", "1"})
+	c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3", "pre4-1"})
 
 	e, u, d = parseVersion("4:1.3-pre4-1")
-	c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3-pre4", "1"})
+	c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3", "pre4-1"})
 }
 
 func (s *VersionSuite) TestCompareLexicographic(c *C) {
@@ -100,6 +100,7 @@ func (s *VersionSuite) TestCompareVersions(c *C) {
 	c.Check(CompareVersions("1.0-133-avc", "1.0"), Equals, 1)
 
 	c.Check(CompareVersions("5.2.0.3", "5.2.0.283"), Equals, -1)
+	c.Check(CompareVersions("4.3.5a", "4.3.5-rc3-1"), Equals, 1)
 }
 
 func (s *VersionSuite) TestParseDependency(c *C) {

debian/NEWS

@@ -0,0 +1,33 @@
+aptly (1.6.0+ds1-1) unstable; urgency=medium
+
+  - aptly-api: configuration file is now /etc/aptly.conf, and `rootDir`
+    defaults to `~/.aptly`
+
+  - aptly-api: default port is 8080, as declared in
+    `/etc/default/aptly-api`
+
+  - aptly: swagger support is disabled, but will be re-enabled after the
+    corresponding golang packages make it to unstable
+  
+  - aptly: default format is yaml, but the old format is still supported 
+    for now
+
+ -- Sebastien Delafond <seb@debian.org>  Sun, 29 Dec 2024 08:46:07 +0100
+
+aptly (1.3.0+ds1-2.2) unstable; urgency=medium
+
+  This version tries to fix the database backwards compatibility,
+  so you don't need to rebuild the database if you upgrade from
+  aptly <= 1.3.0-6.
+
+  However some fields are missing, like created time of a snapshot.
+
+ -- Shengjing Zhu <zhsj@debian.org>  Sat, 13 Apr 2019 23:26:39 +0800
+
+aptly (1.3.0+ds1-2) unstable; urgency=medium
+
+  * The database created by aptly <= 1.3.0-6 is not compatible
+    with greater versions. Users must create a new database.
+    The `aptly db recover` will not fix the issue.
+
+ -- Alexandre Viau <aviau@debian.org>  Fri, 26 Oct 2018 13:20:53 -0400

debian/aptly.conf

@@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu
 # Codename for short PPA url expansion
 ppa_codename: ""
 
+# PPA Base URL (default: launchpad)
+# # ppa_baseurl: http://ppa.launchpad.net
+
 
 # Aptly Server
 ###############
@@ -80,8 +83,9 @@ serve_in_api_mode: false
 # Enable metrics for Prometheus client
 enable_metrics_endpoint: false
 
+# Not implemented in this version.
 # Enable API documentation on /docs
-enable_swagger_endpoint: false
+#enable_swagger_endpoint: false
 
 # OBSOLETE: use via url param ?_async=true
 async_api: false
@@ -92,27 +96,17 @@ async_api: false
 
 # Database backend
 # Type must be one of:
-# * leveldb (default) - local embedded database
-# * etcd - distributed key-value store for multi-node setups
+# * leveldb (default)
+# * etcd
 database_backend:
     type: leveldb
     # Path to leveldb files
     # empty dbPath defaults to `rootDir`/db
     db_path: ""
 
-    # Etcd Configuration Example:
-    # Etcd allows multiple aptly instances to share the same database
-    # for high availability and load balancing scenarios
-    #
     # type: etcd
-    # # URL to etcd server
+    # # URL to db server
     # url: "127.0.0.1:2379"
-    #
-    # # Additional etcd configuration via environment variables:
-    # # APTLY_ETCD_TIMEOUT - Operation timeout (default: 60s)
-    # # APTLY_ETCD_DIAL_TIMEOUT - Connection timeout (default: 60s)
-    # # APTLY_ETCD_KEEPALIVE - Keep-alive timeout (default: 7200s)
-    # # APTLY_ETCD_MAX_MSG_SIZE - Maximum message size in bytes (default: 52428800)
 
 
 # Mirroring
@@ -262,16 +256,6 @@ s3_publish_endpoints:
     #     # Debug (optional)
     #     # Enables detailed request/response dump for each S3 operation
     #     debug: false
-    #     # Concurrent Uploads (optional)
-    #     # Number of concurrent upload workers for S3 operations.
-    #     # * 0 (default) - uploads are performed sequentially
-    #     # * >0 - enables concurrent uploads with specified number of workers
-    #     concurrent_uploads: 0
-    #     # Upload Queue Size (optional)
-    #     # Multiplier for upload queue size (queue_size = concurrent_uploads * upload_queue_size)
-    #     # * 2 (default) - queue holds 2x the number of workers
-    #     # * higher values allow more uploads to be queued before blocking
-    #     upload_queue_size: 2
 
 # Swift Endpoint Support
 #