Compare commits

..

2 Commits

Author SHA1 Message Date
André Roth 594883f1e5 add test workflow 2025-02-15 23:48:59 +01:00
André Roth 55a4872891 add github action for aptly upload 2025-02-15 23:48:59 +01:00
271 changed files with 1440 additions and 6918 deletions
-5
View File
@@ -4,10 +4,6 @@ Fixes #
All new code should be covered with tests, documentation should be updated. CI should pass. All new code should be covered with tests, documentation should be updated. CI should pass.
Also, to speed up things, if you could kindly "Allow edits and access to secrets by maintainers" in the
PR settings, as this allows us to rebase the PR on master, fix conflicts, run coverage and help with
implementing code and tests.
## Description of the Change ## Description of the Change
<!-- <!--
@@ -18,7 +14,6 @@ Why this change is important?
## Checklist ## Checklist
- [ ] allow Maintainers to edit PR (rebase, run coverage, help with tests, ...)
- [ ] unit-test added (if change is algorithm) - [ ] unit-test added (if change is algorithm)
- [ ] functional test added/updated (if change is functional) - [ ] functional test added/updated (if change is functional)
- [ ] man page updated (if applicable) - [ ] man page updated (if applicable)
+28 -99
View File
@@ -17,32 +17,8 @@ env:
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
jobs: jobs:
unit-test:
name: "Unit Tests"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Docker Image"
run: |
make docker-image
- name: "Unit Tests"
run: |
make docker-unit-tests
mkdir -p out/coverage
mv unit.out out/coverage/
- uses: actions/upload-artifact@v4
with:
name: unit-tests-coverage
path: out/
test: test:
name: "System Test" name: "Test (Ubuntu 22.04)"
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
continue-on-error: false continue-on-error: false
timeout-minutes: 30 timeout-minutes: 30
@@ -54,10 +30,10 @@ jobs:
GOPROXY: "https://proxy.golang.org" GOPROXY: "https://proxy.golang.org"
steps: steps:
- name: "Install Test Packages" - name: "Install Packages"
run: | run: |
sudo apt-get update sudo apt-get update
sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
- name: "Checkout Repository" - name: "Checkout Repository"
uses: actions/checkout@v4 uses: actions/checkout@v4
@@ -87,10 +63,21 @@ jobs:
with: with:
directory: ${{ runner.temp }} directory: ${{ runner.temp }}
- name: "Run Unit Tests"
env:
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make test
- name: "Run Benchmark" - name: "Run Benchmark"
run: | run: |
mkdir -p out/coverage COVERAGE_DIR=${{ runner.temp }} make bench
COVERAGE_DIR=$PWD/out/coverage make bench
- name: "Run System Tests" - name: "Run System Tests"
env: env:
@@ -102,63 +89,30 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: | run: |
sudo mkdir -p /srv ; sudo chown runner /srv sudo mkdir -p /srv ; sudo chown runner /srv
mkdir -p out/coverage COVERAGE_DIR=${{ runner.temp }} make system-test
COVERAGE_DIR=$PWD/out/coverage make system-test
- uses: actions/upload-artifact@v4
with:
name: system-tests-coverage
path: out/
coverage:
name: "Upload Coverage"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
needs:
- unit-test
- test
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
- name: "Download Unit Test Coverage"
uses: actions/download-artifact@v4
with:
name: unit-tests-coverage
- name: "Download System Test Coverage"
uses: actions/download-artifact@v4
with:
name: system-tests-coverage
- name: "Merge Code Coverage" - name: "Merge Code Coverage"
run: | run: |
# go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad go install github.com/wadey/gocovmerge@latest
# ~/go/bin/gocovmerge coverage/*.out > coverage.txt ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
- name: "Upload Code Coverage" - name: "Upload Code Coverage"
if: github.actor != 'dependabot[bot]' uses: codecov/codecov-action@v2
uses: codecov/codecov-action@v5
with: with:
token: ${{ secrets.CODECOV_TOKEN }} token: ${{ secrets.CODECOV_TOKEN }}
files: coverage.txt files: coverage.txt
fail_ci_if_error: true
ci-debian-build: ci-debian-build:
name: "Build" name: "Build"
needs: needs: test
- coverage
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"] name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
arch: ["amd64", "i386" , "arm64" , "armhf"] arch: ["amd64", "i386" , "arm64" , "armhf"]
include: include:
- name: "Debian 13/trixie" - name: "Debian 13/testing"
suite: trixie suite: trixie
image: debian:trixie-slim image: debian:trixie-slim
- name: "Debian 12/bookworm" - name: "Debian 12/bookworm"
@@ -167,6 +121,9 @@ jobs:
- name: "Debian 11/bullseye" - name: "Debian 11/bullseye"
suite: bullseye suite: bullseye
image: debian:bullseye-slim image: debian:bullseye-slim
- name: "Debian 10/buster"
suite: buster
image: debian:buster-slim
- name: "Ubuntu 24.04" - name: "Ubuntu 24.04"
suite: noble suite: noble
image: ubuntu:24.04 image: ubuntu:24.04
@@ -178,12 +135,11 @@ jobs:
image: ubuntu:20.04 image: ubuntu:20.04
container: container:
image: ${{ matrix.image }} image: ${{ matrix.image }}
options: --user root
env: env:
APT_LISTCHANGES_FRONTEND: none APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive DEBIAN_FRONTEND: noninteractive
steps: steps:
- name: "Install Build Packages" - name: "Install packages"
run: | run: |
apt-get update apt-get update
apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \ apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
@@ -251,27 +207,9 @@ jobs:
run: | run: |
.github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }} .github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
- name: "Get aptly version"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
aptlyver=$(make -s version)
echo "Aptly Version: $aptlyver"
echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
id: releaseversion
- name: "Upload CI Artifacts"
if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags')
uses: actions/upload-artifact@v4
with:
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }}
path: build/
retention-days: 7
ci-binary-build: ci-binary-build:
name: "Build" name: "Build"
needs: needs: test
- coverage
runs-on: ubuntu-latest runs-on: ubuntu-latest
strategy: strategy:
matrix: matrix:
@@ -329,15 +267,6 @@ jobs:
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression compression-level: 0 # no compression
- name: "Upload CI Artifacts"
uses: actions/upload-artifact@v4
if: "!startsWith(github.event.ref, 'refs/tags')"
with:
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression
retention-days: 7
gh-release: gh-release:
name: "Github Release" name: "Github Release"
runs-on: ubuntu-latest runs-on: ubuntu-latest
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
- name: Install and initialize swagger - name: Install and initialize swagger
run: | run: |
go install github.com/swaggo/swag/cmd/swag@latest go install github.com/swaggo/swag/cmd/swag@latest
swag init -q --propertyStrategy pascalcase --markdownFiles docs swag init -q --markdownFiles docs
shell: sh shell: sh
- name: golangci-lint - name: golangci-lint
@@ -44,7 +44,7 @@ jobs:
# Require: The version of golangci-lint to use. # Require: The version of golangci-lint to use.
# When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
# When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
version: v1.64.5 version: v1.54.1
# Optional: working directory, useful for monorepos # Optional: working directory, useful for monorepos
# working-directory: somedir # working-directory: somedir
+21
View File
@@ -0,0 +1,21 @@
name: test aptly upload-lint
on:
push:
pull_request:
permissions:
contents: read
jobs:
aptly-upload:
name: upload
runs-on: ubuntu-latest
steps:
- name: Publish to aptly server
uses: neolynx/aptly-github-action/publish@v0.0.15
with:
directory: buildout
aptly_server: https://aptly.info
aptly_user: ${{ secrets.APTLY_USER }}
aptly_password: ${{ secrets.APTLY_PASSWORD }}
-2
View File
@@ -26,8 +26,6 @@ _testmain.go
*.test *.test
coverage.txt coverage.txt
coverage.out
coverage.html
*.pyc *.pyc
+15 -10
View File
@@ -1,11 +1,16 @@
version: "2" run:
tests: false
linters: linters:
settings: disable-all: true
staticcheck: enable:
checks: - goconst
- "all" - gofmt
- "-QF1004" # could use strings.ReplaceAll instead - goimports
- "-QF1012" # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...)) - govet
- "-QF1003" # could use tagged switch - ineffassign
- "-ST1000" # at least one file in a package should have a package comment - misspell
- "-QF1001" # could apply De Morgan's law - revive
- staticcheck
- vetshadow
-16
View File
@@ -68,19 +68,3 @@ List of contributors, in chronological order:
* Blake Kostner (https://github.com/btkostner) * Blake Kostner (https://github.com/btkostner)
* Leigh London (https://github.com/leighlondon) * Leigh London (https://github.com/leighlondon)
* Gordian Schoenherr (https://github.com/schoenherrg) * Gordian Schoenherr (https://github.com/schoenherrg)
* Silke Hofstra (https://github.com/silkeh)
* Itay Porezky (https://github.com/itayporezky)
* Alejandro Guijarro Monerris (https://github.com/alguimodd)
* JupiterRider (https://github.com/JupiterRider)
* Agustin Henze (https://github.com/agustinhenze)
* Tobias Assarsson (https://github.com/daedaluz)
* Yaksh Bariya (https://github.com/thunder-coding)
* Juan Calderon-Perez (https://github.com/gaby)
* Ato Araki (https://github.com/atotto)
* Roman Lebedev (https://github.com/LebedevRI)
* Brian Witt (https://github.com/bwitt)
* Ales Bregar (https://github.com/abregar)
* Tim Foerster (https://github.com/tonobo)
* Zhang Xiao (https://github.com/xzhang1)
* Tom Nguyen (https://github.com/lecafard)
* Philip Cramer (https://github.com/PhilipCramer)
+25 -50
View File
@@ -2,32 +2,13 @@ GOPATH=$(shell go env GOPATH)
VERSION=$(shell make -s version) VERSION=$(shell make -s version)
PYTHON?=python3 PYTHON?=python3
BINPATH?=$(GOPATH)/bin BINPATH?=$(GOPATH)/bin
GOLANGCI_LINT_VERSION=v2.0.2 # version supporting go 1.24 GOLANGCI_LINT_VERSION=v1.54.1 # version supporting go 1.19
COVERAGE_DIR?=$(shell mktemp -d) COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS) GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH) GOARCH=$(shell go env GOHOSTARCH)
export PODMAN_USERNS = keep-id # Uncomment to update system test gold files
DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src # CAPTURE := "--capture"
# Setting TZ for certificates
export TZ=UTC
# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
export TEST_FAKETIME := 2025-01-02 03:04:05
# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
ifeq ($(COVERAGE_SKIP),1)
COVERAGE_ARG_BUILD :=
COVERAGE_ARG_TEST := --coverage-skip
else
COVERAGE_ARG_BUILD := -coverpkg="./..."
COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
endif
# export CAPUTRE=1 for regenrating test gold files
ifeq ($(CAPTURE),1)
CAPTURE_ARG := --capture
endif
help: ## Print this help help: ## Print this help
@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -69,7 +50,7 @@ swagger-install:
echo "// @version $(VERSION)" >> docs/swagger.conf echo "// @version $(VERSION)" >> docs/swagger.conf
azurite-start: azurite-start:
azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \ azurite -l /tmp/aptly-azurite & \
echo $$! > ~/.azurite.pid echo $$! > ~/.azurite.pid
azurite-stop: azurite-stop:
@@ -77,7 +58,7 @@ azurite-stop:
swagger: swagger-install swagger: swagger-install
# Generate swagger docs # Generate swagger docs
@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf @PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
etcd-install: etcd-install:
# Install etcd # Install etcd
@@ -88,9 +69,9 @@ flake8: ## run flake8 on system test python files
lint: prepare lint: prepare
# Install golangci-lint # Install golangci-lint
@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION) @test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
# Running lint # Running lint
@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0 @PATH=$(BINPATH)/:$(PATH) golangci-lint run
build: prepare swagger ## Build aptly build: prepare swagger ## Build aptly
@@ -103,11 +84,11 @@ install:
# go install -v # go install -v
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi @out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify which tests to run) test: prepare swagger etcd-install ## Run unit tests
@echo "\e[33m\e[1mStarting etcd ...\e[0m" @echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 & @mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
@echo "\e[33m\e[1mRunning go test ...\e[0m" @echo "\e[33m\e[1mRunning go test ...\e[0m"
faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\e[33m\e[1mStopping etcd ...\e[0m" @echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid @pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/aptly-etcd-data/etcd.log @rm -f /tmp/aptly-etcd-data/etcd.log
@@ -115,13 +96,13 @@ test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify
system-test: prepare swagger etcd-install ## Run system tests system-test: prepare swagger etcd-install ## Run system tests
# build coverage binary # build coverage binary
go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli go test -v -coverpkg="./..." -c -tags testruncli
# Download fixture-db, fixture-pool, etcd.db # Download fixture-db, fixture-pool, etcd.db
if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests # Run system tests
PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST) PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
bench: bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m" @echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -131,8 +112,7 @@ serve: prepare swagger-install ## Run development server (auto recompiling)
test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
cp debian/aptly.conf ~/.aptly.conf cp debian/aptly.conf ~/.aptly.conf
sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
dpkg: prepare swagger ## Build debian packages dpkg: prepare swagger ## Build debian packages
@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
@@ -182,49 +162,44 @@ binaries: prepare swagger ## Build binary releases (FreeBSD, macOS, Linux gener
docker-image: ## Build aptly-dev docker image docker-image: ## Build aptly-dev docker image
@docker build -f system/Dockerfile . -t aptly-dev @docker build -f system/Dockerfile . -t aptly-dev
docker-image-no-cache: ## Build aptly-dev docker image (no cache)
@docker build --no-cache -f system/Dockerfile . -t aptly-dev
docker-build: ## Build aptly in docker container docker-build: ## Build aptly in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
docker-shell: ## Run aptly and other commands in docker container docker-shell: ## Run aptly and other commands in docker container
@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
docker-deb: ## Build debian packages in docker container docker-deb: ## Build debian packages in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
docker-unit-tests: ## Run unit tests in docker container (add TEST=regex to specify which tests to run) docker-unit-test: ## Run unit tests in docker container
$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \ @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \ azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
test TEST=$(TEST) \ test \
azurite-stop azurite-stop
docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \ @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \ azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ system-test TEST=$(TEST) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
azurite-stop azurite-stop
docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 docker-serve: ## Run development server (auto recompiling) on http://localhost:3142
@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
docker-lint: ## Run golangci-lint in docker container docker-lint: ## Run golangci-lint in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
docker-man: ## Create man page in docker container docker-man: ## Create man page in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
mem.png: mem.dat mem.gp mem.png: mem.dat mem.gp
gnuplot mem.gp gnuplot mem.gp
@@ -241,4 +216,4 @@ clean: ## remove local build and module cache
rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
.PHONY: help man prepare swagger version binaries build docker-release docker-system-tests docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8 .PHONY: help man prepare swagger version binaries build docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image docker-man docker-shell docker-serve clean releasetype dpkg serve flake8
+6 -4
View File
@@ -63,7 +63,7 @@ Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble`` Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
Install aptly packages:: Install aptly packages::
@@ -80,7 +80,7 @@ Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble`` Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
Note: same gpg key is used as for the Upstream Debian Packages. Note: same gpg key is used as for the Upstream Debian Packages.
@@ -111,8 +111,10 @@ With configuration management systems:
- `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer - `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
(Heavy Water Operations, LLC) (Heavy Water Operations, LLC)
- `Puppet module <https://github.com/voxpupuli/puppet-aptly>`_ by - `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
Vox Pupuli Government Digital Services
- `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
TubeMogul
- `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by - `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
Forrest Alvarez and Brian Jackson Forrest Alvarez and Brian Jackson
- `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine - `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+1 -3
View File
@@ -12,7 +12,5 @@ git push origin v$version master
``` ```
- run swagger locally (`make docker-serve`) - run swagger locally (`make docker-serve`)
- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json - copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
- add new version to select tag in content/doc/api/swagger.md line 48 - releae www.aptly.info
- update version in content/download.md
- push commit to master
- create release announcement on https://github.com/aptly-dev/aptly/discussions - create release announcement on https://github.com/aptly-dev/aptly/discussions
+17 -21
View File
@@ -41,10 +41,7 @@ type aptlyVersion struct {
// @Success 200 {object} aptlyVersion // @Success 200 {object} aptlyVersion
// @Router /api/version [get] // @Router /api/version [get]
func apiVersion(c *gin.Context) { func apiVersion(c *gin.Context) {
version := aptlyVersion{ c.JSON(200, gin.H{"Version": aptly.Version})
Version: aptly.Version,
}
c.JSON(200, version)
} }
type aptlyStatus struct { type aptlyStatus struct {
@@ -70,8 +67,7 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) {
return return
} }
status := aptlyStatus{Status: "Aptly is ready"} c.JSON(200, gin.H{"Status": "Aptly is ready"})
c.JSON(200, status)
} }
} }
@@ -169,7 +165,7 @@ func runTaskInBackground(name string, resources []string, proc task.Process) (ta
return nil, err return nil, err
} }
defer func() { _ = releaseDatabaseConnection() }() defer releaseDatabaseConnection()
return proc(out, detail) return proc(out, detail)
}) })
} }
@@ -178,18 +174,18 @@ func truthy(value interface{}) bool {
if value == nil { if value == nil {
return false return false
} }
switch v := value.(type) { switch value.(type) {
case string: case string:
switch strings.ToLower(v) { switch strings.ToLower(value.(string)) {
case "n", "no", "f", "false", "0", "off": case "n", "no", "f", "false", "0", "off":
return false return false
default: default:
return true return true
} }
case int: case int:
return v != 0 return !(value.(int) == 0)
case bool: case bool:
return v return value.(bool)
} }
return true return true
} }
@@ -214,11 +210,11 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
} }
// wait for task to finish // wait for task to finish
_, _ = context.TaskList().WaitForTaskByID(task.ID) context.TaskList().WaitForTaskByID(task.ID)
retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID) retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
err, _ := context.TaskList().GetTaskErrorByID(task.ID) err, _ := context.TaskList().GetTaskErrorByID(task.ID)
_, _ = context.TaskList().DeleteTaskByID(task.ID) context.TaskList().DeleteTaskByID(task.ID)
if err != nil { if err != nil {
AbortWithJSONError(c, retValue.Code, err) AbortWithJSONError(c, retValue.Code, err)
return return
@@ -286,11 +282,11 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
// filter packages by version // filter packages by version
if c.Request.URL.Query().Get("maximumVersion") == "1" { if c.Request.URL.Query().Get("maximumVersion") == "1" {
list.PrepareIndex() list.PrepareIndex()
_ = list.ForEach(func(p *deb.Package) error { list.ForEach(func(p *deb.Package) error {
versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version)) versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
if err != nil { if err != nil {
fmt.Println("filter packages by version, query string parse err: ", err) fmt.Println("filter packages by version, query string parse err: ", err)
_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err)) c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
} else { } else {
tmpList, err := list.Filter(deb.FilterOptions{ tmpList, err := list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{versionQ}, Queries: []deb.PackageQuery{versionQ},
@@ -298,15 +294,15 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
if err == nil { if err == nil {
if tmpList.Len() > 0 { if tmpList.Len() > 0 {
_ = tmpList.ForEach(func(tp *deb.Package) error { tmpList.ForEach(func(tp *deb.Package) error {
list.Remove(tp) list.Remove(tp)
return nil return nil
}) })
_ = list.Add(p) list.Add(p)
} }
} else { } else {
fmt.Println("filter packages by version, filter err: ", err) fmt.Println("filter packages by version, filter err: ", err)
_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err)) c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
} }
} }
@@ -315,7 +311,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
} }
if c.Request.URL.Query().Get("format") == "details" { if c.Request.URL.Query().Get("format") == "details" {
_ = list.ForEach(func(p *deb.Package) error { list.ForEach(func(p *deb.Package) error {
result = append(result, p) result = append(result, p)
return nil return nil
}) })
@@ -326,7 +322,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
} }
} }
func AbortWithJSONError(c *gin.Context, code int, err error) { func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8") c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
_ = c.AbortWithError(code, err) return c.AbortWithError(code, err)
} }
+17 -17
View File
@@ -24,14 +24,14 @@ func Test(t *testing.T) {
TestingT(t) TestingT(t)
} }
type APISuite struct { type ApiSuite struct {
context *ctx.AptlyContext context *ctx.AptlyContext
flags *flag.FlagSet flags *flag.FlagSet
configFile *os.File configFile *os.File
router http.Handler router http.Handler
} }
var _ = Suite(&APISuite{}) var _ = Suite(&ApiSuite{})
func createTestConfig() *os.File { func createTestConfig() *os.File {
file, err := os.CreateTemp("", "aptly") file, err := os.CreateTemp("", "aptly")
@@ -45,11 +45,11 @@ func createTestConfig() *os.File {
if err != nil { if err != nil {
return nil return nil
} }
_, _ = file.Write(jsonString) file.Write(jsonString)
return file return file
} }
func (s *APISuite) setupContext() error { func (s *ApiSuite) setupContext() error {
aptly.Version = "testVersion" aptly.Version = "testVersion"
file := createTestConfig() file := createTestConfig()
if nil == file { if nil == file {
@@ -75,23 +75,23 @@ func (s *APISuite) setupContext() error {
return nil return nil
} }
func (s *APISuite) SetUpSuite(c *C) { func (s *ApiSuite) SetUpSuite(c *C) {
err := s.setupContext() err := s.setupContext()
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
func (s *APISuite) TearDownSuite(c *C) { func (s *ApiSuite) TearDownSuite(c *C) {
_ = os.Remove(s.configFile.Name()) os.Remove(s.configFile.Name())
s.context.Shutdown() s.context.Shutdown()
} }
func (s *APISuite) SetUpTest(c *C) { func (s *ApiSuite) SetUpTest(c *C) {
} }
func (s *APISuite) TearDownTest(c *C) { func (s *ApiSuite) TearDownTest(c *C) {
} }
func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) { func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
w := httptest.NewRecorder() w := httptest.NewRecorder()
req, err := http.NewRequest(method, url, body) req, err := http.NewRequest(method, url, body)
if err != nil { if err != nil {
@@ -102,32 +102,32 @@ func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*http
return w, nil return w, nil
} }
func (s *APISuite) TestGinRunsInReleaseMode(c *C) { func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
c.Check(gin.Mode(), Equals, gin.ReleaseMode) c.Check(gin.Mode(), Equals, gin.ReleaseMode)
} }
func (s *APISuite) TestGetVersion(c *C) { func (s *ApiSuite) TestGetVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/version", nil) response, err := s.HTTPRequest("GET", "/api/version", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200) c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}") c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
} }
func (s *APISuite) TestGetReadiness(c *C) { func (s *ApiSuite) TestGetReadiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/ready", nil) response, err := s.HTTPRequest("GET", "/api/ready", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200) c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}") c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
} }
func (s *APISuite) TestGetHealthiness(c *C) { func (s *ApiSuite) TestGetHealthiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/healthy", nil) response, err := s.HTTPRequest("GET", "/api/healthy", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200) c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}") c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
} }
func (s *APISuite) TestGetMetrics(c *C) { func (s *ApiSuite) TestGetMetrics(c *C) {
response, err := s.HTTPRequest("GET", "/api/metrics", nil) response, err := s.HTTPRequest("GET", "/api/metrics", nil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200) c.Check(response.Code, Equals, 200)
@@ -141,7 +141,7 @@ func (s *APISuite) TestGetMetrics(c *C) {
c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*") c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
} }
func (s *APISuite) TestRepoCreate(c *C) { func (s *ApiSuite) TestRepoCreate(c *C) {
body, err := json.Marshal(gin.H{ body, err := json.Marshal(gin.H{
"Name": "dummy", "Name": "dummy",
}) })
@@ -150,7 +150,7 @@ func (s *APISuite) TestRepoCreate(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
func (s *APISuite) TestTruthy(c *C) { func (s *ApiSuite) TestTruthy(c *C) {
c.Check(truthy("no"), Equals, false) c.Check(truthy("no"), Equals, false)
c.Check(truthy("n"), Equals, false) c.Check(truthy("n"), Equals, false)
c.Check(truthy("off"), Equals, false) c.Check(truthy("off"), Equals, false)
+3 -3
View File
@@ -21,7 +21,7 @@ import (
// @Success 200 {object} string "Output" // @Success 200 {object} string "Output"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Router /api/db/cleanup [post] // @Router /api/db/cleanup [post]
func apiDBCleanup(c *gin.Context) { func apiDbCleanup(c *gin.Context) {
resources := []string{string(task.AllResourcesKey)} resources := []string{string(task.AllResourcesKey)}
maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
var err error var err error
@@ -109,8 +109,8 @@ func apiDBCleanup(c *gin.Context) {
if toDelete.Len() > 0 { if toDelete.Len() > 0 {
batch := db.CreateBatch() batch := db.CreateBatch()
_ = toDelete.ForEach(func(ref []byte) error { toDelete.ForEach(func(ref []byte) error {
_ = collectionFactory.PackageCollection().DeleteByKey(ref, batch) collectionFactory.PackageCollection().DeleteByKey(ref, batch)
return nil return nil
}) })
+2 -41
View File
@@ -13,10 +13,6 @@ import (
"github.com/saracen/walker" "github.com/saracen/walker"
) )
// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
// In production it calls (*os.File).Sync().
var syncFile = func(f *os.File) error { return f.Sync() }
func verifyPath(path string) bool { func verifyPath(path string) bool {
path = filepath.Clean(path) path = filepath.Clean(path)
for _, part := range strings.Split(path, string(filepath.Separator)) { for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -118,69 +114,34 @@ func apiFilesUpload(c *gin.Context) {
} }
stored := []string{} stored := []string{}
openFiles := []*os.File{}
// Write all files first
for _, files := range c.Request.MultipartForm.File { for _, files := range c.Request.MultipartForm.File {
for _, file := range files { for _, file := range files {
src, err := file.Open() src, err := file.Open()
if err != nil { if err != nil {
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err) AbortWithJSONError(c, 500, err)
return return
} }
defer src.Close()
destPath := filepath.Join(path, filepath.Base(file.Filename)) destPath := filepath.Join(path, filepath.Base(file.Filename))
dst, err := os.Create(destPath) dst, err := os.Create(destPath)
if err != nil { if err != nil {
_ = src.Close()
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err) AbortWithJSONError(c, 500, err)
return return
} }
defer dst.Close()
_, err = io.Copy(dst, src) _, err = io.Copy(dst, src)
_ = src.Close()
if err != nil { if err != nil {
_ = dst.Close()
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err) AbortWithJSONError(c, 500, err)
return return
} }
// Keep file open for batch sync
openFiles = append(openFiles, dst)
stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename))) stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
} }
} }
// Sync all files at once to catch ENOSPC errors
for i, dst := range openFiles {
err := syncFile(dst)
if err != nil {
// Close all files
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
return
}
}
// Close all files
for _, dst := range openFiles {
_ = dst.Close()
}
apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc() apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
c.JSON(200, stored) c.JSON(200, stored)
} }
-476
View File
@@ -1,476 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"github.com/smira/flag"
. "gopkg.in/check.v1"
)
type FilesUploadDiskFullSuite struct {
aptlyContext *ctx.AptlyContext
flags *flag.FlagSet
configFile *os.File
router http.Handler
}
var _ = Suite(&FilesUploadDiskFullSuite{})
func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
aptly.Version = "testVersion"
file, err := os.CreateTemp("", "aptly")
c.Assert(err, IsNil)
s.configFile = file
jsonString, err := json.Marshal(gin.H{
"architectures": []string{},
"rootDir": c.MkDir(),
})
c.Assert(err, IsNil)
_, err = file.Write(jsonString)
c.Assert(err, IsNil)
_ = file.Close()
flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
flags.Bool("no-lock", false, "dummy")
flags.Int("db-open-attempts", 3, "dummy")
flags.String("config", s.configFile.Name(), "dummy")
flags.String("architectures", "", "dummy")
s.flags = flags
aptlyContext, err := ctx.NewContext(s.flags)
c.Assert(err, IsNil)
s.aptlyContext = aptlyContext
s.router = Router(aptlyContext)
context = aptlyContext
}
func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
if s.configFile != nil {
_ = os.Remove(s.configFile.Name())
}
if s.aptlyContext != nil {
s.aptlyContext.Shutdown()
}
}
func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
testContent := []byte("test file content for upload")
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "testfile.txt")
c.Assert(err, IsNil)
_, err = part.Write(testContent)
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil)
c.Check(content, DeepEquals, testContent)
}
func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
testContent := bytes.Repeat([]byte("A"), 10000)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "largefile.bin")
c.Assert(err, IsNil)
_, err = io.Copy(part, bytes.NewReader(testContent))
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir2", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil)
c.Check(len(content), Equals, len(testContent))
c.Check(content, DeepEquals, testContent)
}
func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
testFiles := map[string][]byte{
"file1.txt": []byte("content of file 1"),
"file2.txt": bytes.Repeat([]byte("B"), 5000),
"file3.deb": []byte("debian package content"),
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
for filename, content := range testFiles {
part, err := writer.CreateFormFile("file", filename)
c.Assert(err, IsNil)
_, err = part.Write(content)
c.Assert(err, IsNil)
}
err := writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/multitest", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
for filename, expectedContent := range testFiles {
uploadedFile := filepath.Join(uploadDir, filename)
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
}
}
func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
oldSyncFile := syncFile
syncFile = func(f *os.File) error {
if filepath.Base(f.Name()) == "syncfail.txt" {
return syscall.ENOSPC
}
return nil
}
defer func() { syncFile = oldSyncFile }()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part1, err := writer.CreateFormFile("file", "ok.txt")
c.Assert(err, IsNil)
_, err = part1.Write([]byte("ok"))
c.Assert(err, IsNil)
part2, err := writer.CreateFormFile("file", "syncfail.txt")
c.Assert(err, IsNil)
_, err = part2.Write([]byte("will fail on sync"))
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
c.Check(verifyPath("a/b/c"), Equals, true)
c.Check(verifyPath("../x"), Equals, false)
c.Check(verifyPath("./x"), Equals, true)
c.Check(verifyPath(".."), Equals, false)
c.Check(verifyPath("."), Equals, false)
}
func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
}
func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
uploadRoot := s.aptlyContext.UploadPath()
c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
body := w.Body.String()
c.Check(strings.Contains(body, "d1"), Equals, true)
c.Check(strings.Contains(body, "d2"), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 404)
}
func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files/dir", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
body := w.Body.String()
c.Check(strings.Contains(body, "a.txt"), Equals, true)
c.Check(strings.Contains(body, "b.txt"), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
_, statErr := os.Stat(base)
c.Check(os.IsNotExist(statErr), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
_, statErr := os.Stat(filepath.Join(base, "a.txt"))
c.Check(os.IsNotExist(statErr), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
c.Assert(os.MkdirAll(base, 0777), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
}
func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
req, err := http.NewRequest("DELETE", "/api/files/..", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
c.Assert(os.MkdirAll(base, 0777), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
}
func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.Chmod(base, 0), IsNil)
defer func() { _ = os.Chmod(base, 0777) }()
req, err := http.NewRequest("GET", "/api/files/noperms", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
c.Assert(os.MkdirAll(base, 0777), IsNil)
subdir := filepath.Join(base, "subdir")
c.Assert(os.MkdirAll(subdir, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
c.Assert(err, IsNil)
req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/..", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
form := &multipart.Form{
File: map[string][]*multipart.FileHeader{
"file": {{Filename: "broken.bin"}},
},
}
req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
c.Assert(err, IsNil)
req.MultipartForm = form
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.Chmod(base, 0555), IsNil)
defer func() { _ = os.Chmod(base, 0777) }()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/readonly", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
parent := s.aptlyContext.UploadPath()
base := filepath.Join(parent, "cantremove")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.Chmod(parent, 0555), IsNil)
defer func() { _ = os.Chmod(parent, 0777) }()
req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
+9 -218
View File
@@ -1,7 +1,6 @@
package api package api
import ( import (
"bufio"
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
@@ -13,60 +12,27 @@ import (
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
) )
type gpgKeyInfo struct {
// 16-character key ID (short form)
KeyID string `json:"KeyID" example:"8B48AD6246925553"`
// Full fingerprint
Fingerprint string `json:"Fingerprint" example:"D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0"`
// Key validity (u=unknown, f=fulltrust, m=marginal, n=never)
Validity string `json:"Validity" example:"u"`
// User ID(s) associated with this key
UserIDs []string `json:"UserIDs" example:"John Doe <john@example.com>"`
// Creation date (Unix timestamp format from gpg)
CreatedAt string `json:"CreatedAt" example:"2023-01-15"`
}
type gpgKeyListResponse struct {
Keys []gpgKeyInfo `json:"Keys"`
}
type gpgAddKeyParams struct { type gpgAddKeyParams struct {
// Keyserver, when downloading GpgKeyIDs
Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"`
// GpgKeyIDs to download from Keyserver, comma separated list
GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500,8B48AD6246925553"`
// Armored gpg public ket, instead of downloading from keyserver
GpgKeyArmor string `json:"GpgKeyArmor" example:""`
// Keyring for adding the keys (default: trustedkeys.gpg) // Keyring for adding the keys (default: trustedkeys.gpg)
Keyring string `json:"Keyring" example:"trustedkeys.gpg"` Keyring string `json:"Keyring" example:"trustedkeys.gpg"`
// Add ASCII armored gpg public key, do not download from keyserver
GpgKeyArmor string `json:"GpgKeyArmor" example:""`
// Keyserver to download keys provided in `GpgKeyID`
Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"`
// Keys do download from `Keyserver`, separated by space
GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500 8B48AD6246925553"`
}
type gpgDeleteKeyParams struct {
// Keyring to delete keys from (default: trustedkeys.gpg)
Keyring string `json:"Keyring" example:"trustedkeys.gpg"`
// Key ID or fingerprint to delete
GpgKeyID string `json:"GpgKeyID" example:"8B48AD6246925553"`
} }
// @Summary Add GPG Keys // @Summary Add GPG Keys
// @Description **Adds GPG keys to aptly keyring** // @Description **Adds GPG keys to aptly keyring**
// @Description // @Description
// @Description Add GPG public keys for veryfing remote repositories for mirroring. // @Description Add GPG public keys for veryfing remote repositories for mirroring.
// @Description
// @Description Keys can be added in two ways:
// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
// @Description
// @Tags Mirrors // @Tags Mirrors
// @Consume json
// @Param request body gpgAddKeyParams true "Parameters"
// @Produce json // @Produce json
// @Success 200 {object} string "OK" // @Success 200 {object} string "OK"
// @Failure 400 {object} Error "Bad Request" // @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/key [post] // @Failure 404 {object} Error "Not Found"
// @Router /api/gpg [post]
func apiGPGAddKey(c *gin.Context) { func apiGPGAddKey(c *gin.Context) {
b := gpgAddKeyParams{} b := gpgAddKeyParams{}
if c.Bind(&b) != nil { if c.Bind(&b) != nil {
@@ -94,7 +60,7 @@ func apiGPGAddKey(c *gin.Context) {
AbortWithJSONError(c, 400, err) AbortWithJSONError(c, 400, err)
return return
} }
defer func() { _ = os.RemoveAll(tempdir) }() defer os.RemoveAll(tempdir)
keypath := filepath.Join(tempdir, "key") keypath := filepath.Join(tempdir, "key")
keyfile, e := os.Create(keypath) keyfile, e := os.Create(keypath)
@@ -134,178 +100,3 @@ func apiGPGAddKey(c *gin.Context) {
c.JSON(200, string(out)) c.JSON(200, string(out))
} }
// @Summary List GPG Keys
// @Description **Lists all GPG keys in aptly keyring**
// @Description
// @Description Returns all public keys currently installed in the aptly GPG keyring.
// @Description
// @Tags Mirrors
// @Param keyring query string false "Keyring file to list keys from (default: trustedkeys.gpg)" example(trustedkeys.gpg)
// @Produce json
// @Success 200 {object} gpgKeyListResponse "OK"
// @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/keys [get]
func apiGPGListKeys(c *gin.Context) {
keyring := c.DefaultQuery("keyring", "trustedkeys.gpg")
keyring = utils.SanitizePath(keyring)
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
AbortWithJSONError(c, 400, err)
return
}
args := []string{
"--no-default-keyring",
"--with-colons",
"--keyring", keyring,
"--list-keys",
}
cmd := exec.Command(gpg, args...)
out, err := cmd.CombinedOutput()
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("failed to list keys: %s", string(out)))
return
}
keys := parseGPGOutput(string(out))
c.JSON(200, gpgKeyListResponse{Keys: keys})
}
// @Summary Delete GPG Key
// @Description **Deletes a GPG key from aptly keyring**
// @Description
// @Description Removes a public key from the aptly GPG keyring. This is useful for removing
// @Description compromised keys or cleaning up obsolete keys.
// @Description
// @Tags Mirrors
// @Consume json
// @Param request body gpgDeleteKeyParams true "Parameters"
// @Produce json
// @Success 200 {object} string "OK"
// @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/key [delete]
func apiGPGDeleteKey(c *gin.Context) {
b := gpgDeleteKeyParams{}
if c.Bind(&b) != nil {
AbortWithJSONError(c, 400, fmt.Errorf("invalid request body"))
return
}
if len(strings.TrimSpace(b.GpgKeyID)) == 0 {
AbortWithJSONError(c, 400, fmt.Errorf("GpgKeyID is required"))
return
}
b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
// b.Keyring can be an absolute path
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
AbortWithJSONError(c, 400, err)
return
}
args := []string{
"--no-default-keyring",
"--allow-non-selfsigned-uid",
"--batch",
"--yes",
}
keyring := "trustedkeys.gpg"
if len(b.Keyring) > 0 {
keyring = b.Keyring
}
args = append(args, "--keyring", keyring)
args = append(args, "--delete-keys", b.GpgKeyID)
cmd := exec.Command(gpg, args...)
fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
out, err := cmd.CombinedOutput()
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("failed to delete key: %s", string(out)))
return
}
c.JSON(200, string(out))
}
// parseGPGOutput parses the output of `gpg --with-colons --list-keys`
// and returns a structured list of keys
func parseGPGOutput(output string) []gpgKeyInfo {
var keys []gpgKeyInfo
var currentKey *gpgKeyInfo
scanner := bufio.NewScanner(strings.NewReader(output))
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
parts := strings.Split(line, ":")
if len(parts) < 10 {
continue
}
recordType := parts[0]
// pub: public key record
if recordType == "pub" {
// Save previous key if it exists
if currentKey != nil && currentKey.KeyID != "" {
keys = append(keys, *currentKey)
}
// Create new key entry
// Format: pub:trust:length:algo:keyid:created:expires:uidhash:...
keyID := parts[4]
if len(keyID) >= 16 {
keyID = keyID[len(keyID)-16:] // Last 16 chars = short key ID
}
validity := parts[1]
createdAt := parts[5]
currentKey = &gpgKeyInfo{
KeyID: keyID,
Validity: validity,
CreatedAt: createdAt,
UserIDs: []string{},
Fingerprint: "",
}
}
// uid: user ID record
if recordType == "uid" && currentKey != nil {
// Format: uid:trust:created:expires:keyid:uidhash:uidtype:validity:userID:...
if len(parts) >= 10 {
userID := parts[9]
if userID != "" {
currentKey.UserIDs = append(currentKey.UserIDs, userID)
}
}
}
// fpr: fingerprint record
if recordType == "fpr" && currentKey != nil {
// Format: fpr:::::::::fingerprint:
if len(parts) >= 10 {
fingerprint := parts[9]
currentKey.Fingerprint = fingerprint
}
}
}
// Don't forget the last key
if currentKey != nil && currentKey.KeyID != "" {
keys = append(keys, *currentKey)
}
return keys
}
-451
View File
@@ -1,451 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
. "gopkg.in/check.v1"
)
type GPGSuite struct {
APISuite
}
var _ = Suite(&GPGSuite{})
func (s *GPGSuite) withFakeGPG(c *C, scriptBody string, test func(scriptPath string)) {
tempDir, err := os.MkdirTemp("", "aptly-fake-gpg")
c.Assert(err, IsNil)
defer func() { _ = os.RemoveAll(tempDir) }()
scriptPath := filepath.Join(tempDir, "gpg")
err = os.WriteFile(scriptPath, []byte(scriptBody), 0o755)
c.Assert(err, IsNil)
oldPath := os.Getenv("PATH")
err = os.Setenv("PATH", tempDir+string(os.PathListSeparator)+oldPath)
c.Assert(err, IsNil)
defer func() { _ = os.Setenv("PATH", oldPath) }()
test(scriptPath)
}
func (s *GPGSuite) fakeGPGScript(c *C, listOutput string, deleteOutput string, deleteError string) string {
return "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then\n" +
" echo 'gpg (GnuPG) 2.2.27'\n" +
" exit 0\n" +
"fi\n" +
"args=\"$*\"\n" +
"if printf '%s' \"$args\" | grep -q -- '--list-keys'; then\n" +
" cat <<'EOF'\n" + listOutput + "\nEOF\n" +
" exit 0\n" +
"fi\n" +
"if printf '%s' \"$args\" | grep -q -- '--delete-keys'; then\n" +
" if [ -n \"" + strings.ReplaceAll(deleteError, "\n", "") + "\" ]; then\n" +
" echo '" + strings.ReplaceAll(deleteError, "'", "'\\''") + "'\n" +
" exit 1\n" +
" fi\n" +
" cat <<'EOF'\n" + deleteOutput + "\nEOF\n" +
" exit 0\n" +
"fi\n" +
"echo 'unexpected invocation' >&2\n" +
"exit 1\n"
}
// TestParseGPGOutputEmpty tests parsing of empty GPG output
func (s *GPGSuite) TestParseGPGOutputEmpty(c *C) {
output := ""
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 0)
}
// TestParseGPGOutputSingleKeyMinimal tests parsing a single key with minimal fields
func (s *GPGSuite) TestParseGPGOutputSingleKeyMinimal(c *C) {
// Minimal valid GPG output with one key
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.KeyID, Equals, "8B48AD6246925553")
c.Check(key.Validity, Equals, "u")
c.Check(key.CreatedAt, Equals, "1611864000")
c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
c.Check(key.UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
}
// TestParseGPGOutputMultipleKeys tests parsing multiple keys
func (s *GPGSuite) TestParseGPGOutputMultipleKeys(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
pub:f:2048:1:A1B2C3D4E5F67890:1580592000:1612128000:uidhash:::scESC:::::::23::0:
uid:f::::1580592000::0987654321::Jane Smith <jane@example.com>::::::::::0:
fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 2)
// First key
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
c.Check(keys[0].Validity, Equals, "u")
c.Check(keys[0].UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
// Second key
c.Check(keys[1].KeyID, Equals, "A1B2C3D4E5F67890")
c.Check(keys[1].Validity, Equals, "f")
c.Check(keys[1].UserIDs, DeepEquals, []string{"Jane Smith <jane@example.com>"})
}
// TestParseGPGOutputMultipleUIDs tests a key with multiple user IDs
func (s *GPGSuite) TestParseGPGOutputMultipleUIDs(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
uid:u::::1611864000::1234567891::John Doe <john.doe@company.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.UserIDs, HasLen, 2)
c.Check(key.UserIDs, DeepEquals, []string{
"John Doe <john@example.com>",
"John Doe <john.doe@company.com>",
})
}
// TestParseGPGOutputMalformedLines tests that malformed lines are skipped
func (s *GPGSuite) TestParseGPGOutputMalformedLines(c *C) {
// Mix of valid and invalid lines (too few fields)
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
invalid:line:with:only:three:fields
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
}
// TestParseGPGOutputEmptyLines tests that empty lines are skipped
func (s *GPGSuite) TestParseGPGOutputEmptyLines(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
}
// TestParseGPGOutputKeyWithoutUID tests a public key without user ID
func (s *GPGSuite) TestParseGPGOutputKeyWithoutUID(c *C) {
// Key without uid record (should still be included)
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.KeyID, Equals, "8B48AD6246925553")
c.Check(key.UserIDs, HasLen, 0)
c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
}
// TestParseGPGOutputVariousValidity tests different validity values
func (s *GPGSuite) TestParseGPGOutputVariousValidity(c *C) {
output := `pub:u:4096:1:KEY1111111111111:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::Key1::::::::::0:
fpr:::::::::1111111111111111111111111111111111111111:
pub:f:4096:1:KEY2222222222222:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:f::::1611864000::1234567891::Key2::::::::::0:
fpr:::::::::2222222222222222222222222222222222222222:
pub:m:4096:1:KEY3333333333333:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:m::::1611864000::1234567892::Key3::::::::::0:
fpr:::::::::3333333333333333333333333333333333333333:
pub:n:4096:1:KEY4444444444444:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:n::::1611864000::1234567893::Key4::::::::::0:
fpr:::::::::4444444444444444444444444444444444444444:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 4)
validities := []string{"u", "f", "m", "n"}
for i, validity := range validities {
c.Check(keys[i].Validity, Equals, validity)
}
}
// TestParseGPGOutputShortKeyID tests that key IDs are shortened to 16 chars
func (s *GPGSuite) TestParseGPGOutputShortKeyID(c *C) {
// 40-character key ID that should be shortened to last 16 chars
longKeyID := "0123456789ABCDEF0123456789ABCDEF8B48AD62"
output := `pub:u:4096:1:` + longKeyID + `:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should extract the last 16 characters: 89ABCDEF8B48AD62
c.Check(keys[0].KeyID, Equals, "89ABCDEF8B48AD62")
}
// TestParseGPGOutputSpecialCharactersInUID tests user IDs with special characters
func (s *GPGSuite) TestParseGPGOutputSpecialCharactersInUID(c *C) {
// UID with Unicode characters and special formatting
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::J\xc3\xb6hn D\xc3\xb6\xc3\xa9 (D\xc3\xbcss) <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should preserve the encoded special characters
c.Check(keys[0].UserIDs, HasLen, 1)
}
// TestAPIGPGListKeysDefaultKeyring tests the HTTP endpoint with default keyring
func (s *GPGSuite) TestAPIGPGListKeysDefaultKeyring(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, `pub:u:4096:1:8B48AD6246925553:1611864000:::::
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`, "", ""), func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var result gpgKeyListResponse
err = json.NewDecoder(response.Body).Decode(&result)
c.Assert(err, IsNil)
c.Check(result.Keys, HasLen, 1)
c.Check(result.Keys[0].KeyID, Equals, "8B48AD6246925553")
})
}
// TestAPIGPGListKeysWithKeyringParam tests the HTTP endpoint with custom keyring parameter
func (s *GPGSuite) TestAPIGPGListKeysWithKeyringParam(c *C) {
argFile, err := os.CreateTemp("", "aptly-gpg-args")
c.Assert(err, IsNil)
_ = argFile.Close()
defer func() { _ = os.Remove(argFile.Name()) }()
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
"cat <<'EOF'\n" +
"pub:u:4096:1:8B48AD6246925553:1611864000:::::\n" +
"fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:\n" +
"EOF\n" +
"exit 0\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
response, reqErr := s.HTTPRequest("GET", "/api/gpg/keys?keyring=/custom.gpg", nil)
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 200)
argBytes, readErr := os.ReadFile(argFile.Name())
c.Assert(readErr, IsNil)
c.Check(string(argBytes), Matches, `(?s).*--keyring\ncustom\.gpg\n.*`)
})
}
// TestAPIGPGListKeysResponseFormat tests that the response has the correct structure
func (s *GPGSuite) TestAPIGPGListKeysResponseFormat(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, `pub:f:4096:1:A1B2C3D4E5F67890:1611864000:::::
uid:f::::1611864000::1234567890::Jane Smith <jane@example.com>::::::::::0:
fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`, "", ""), func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var result gpgKeyListResponse
err = json.NewDecoder(response.Body).Decode(&result)
c.Assert(err, IsNil)
c.Assert(result.Keys, HasLen, 1)
c.Check(result.Keys[0].KeyID, Equals, "A1B2C3D4E5F67890")
c.Check(result.Keys[0].Validity, Equals, "f")
c.Check(result.Keys[0].CreatedAt, Equals, "1611864000")
})
}
// TestParseGPGOutputEdgeCaseUIDWithoutFields tests UID record with missing fields
func (s *GPGSuite) TestParseGPGOutputEdgeCaseUIDWithoutFields(c *C) {
// UID record with fewer than 10 fields
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should not have user ID since it's in field 9 and this record is too short
c.Check(keys[0].UserIDs, HasLen, 0)
}
// TestParseGPGOutputFingerprintWithoutCurrentKey tests FPR record appearing before any PUB
func (s *GPGSuite) TestParseGPGOutputFingerprintWithoutCurrentKey(c *C) {
// FPR record without a preceding PUB (should be ignored)
output := `fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should only have one key with the correct fingerprint
c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
}
// TestParseGPGOutputComplexRealWorldExample tests real-world-like GPG output
func (s *GPGSuite) TestParseGPGOutputComplexRealWorldExample(c *C) {
// Real-world GPG output with multiple keys, UIDs, and other record types (sig, sub)
// Note: sub and sig records are skipped as we only care about pub/uid/fpr
realWorldOutput := `tru::1:1611864000:0:3:1:5
pub:u:4096:1:8B48AD6246925553:1611864000:2023-01-15T00:00:00:::::scESC:::::::23::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
uid:u::::1611864100::1234567891::John Doe <john@work.com>::::::::::0:
pub:f:2048:1:1234567890123456:1580592000:2022-12-31T00:00:00::u:::scESC:::::::23::0:
fpr:::::::::F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9:
uid:f::::1580592000::0987654321::Maintainer Key <maint@example.com>::::::::::0:`
keys := parseGPGOutput(realWorldOutput)
c.Check(keys, HasLen, 2)
// First key should have 2 UIDs
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
c.Check(keys[0].UserIDs, HasLen, 2)
c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
// Second key should have 1 UID
c.Check(keys[1].KeyID, Equals, "1234567890123456")
c.Check(keys[1].UserIDs, HasLen, 1)
c.Check(keys[1].Fingerprint, Equals, "F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9")
}
// TestParseGPGOutputConsecutiveEmptyUIDs tests handling of consecutive empty user ID fields
func (s *GPGSuite) TestParseGPGOutputConsecutiveEmptyUIDs(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890:::::::::::0:
uid:u::::1611864000::1234567891::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should skip empty UID but include the non-empty one
c.Check(keys[0].UserIDs, HasLen, 1)
c.Check(keys[0].UserIDs[0], Equals, "John Doe <john@example.com>")
}
// TestGPGDeleteKeyParamsValidation tests gpgDeleteKeyParams validation
func (s *GPGSuite) TestGPGDeleteKeyParamsValidation(c *C) {
// This is a unit test that validates parameter structure (no HTTP needed)
params := gpgDeleteKeyParams{
Keyring: "custom.gpg",
GpgKeyID: "8B48AD6246925553",
}
c.Check(params.Keyring, Equals, "custom.gpg")
c.Check(params.GpgKeyID, Equals, "8B48AD6246925553")
}
// TestAPIGPGDeleteKeyMissingKeyID tests delete with missing key ID parameter
func (s *GPGSuite) TestAPIGPGDeleteKeyMissingKeyID(c *C) {
body, err := json.Marshal(map[string]string{
"Keyring": "trustedkeys.gpg",
// GpgKeyID is missing
})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
}
// TestAPIGPGDeleteKeyInvalidJSON tests delete with invalid JSON request
func (s *GPGSuite) TestAPIGPGDeleteKeyInvalidJSON(c *C) {
response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader([]byte("invalid json")))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
}
// TestAPIGPGDeleteKeySuccess tests successful key deletion
func (s *GPGSuite) TestAPIGPGDeleteKeySuccess(c *C) {
argFile, err := os.CreateTemp("", "aptly-gpg-delete-args")
c.Assert(err, IsNil)
_ = argFile.Close()
defer func() { _ = os.Remove(argFile.Name()) }()
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
"if printf '%s' \"$*\" | grep -q -- '--delete-keys'; then\n" +
"echo 'deleted'\n" +
"exit 0\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
body, marshalErr := json.Marshal(gpgDeleteKeyParams{
Keyring: "/trustedkeys.gpg",
GpgKeyID: "8B48AD6246925553",
})
c.Assert(marshalErr, IsNil)
response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, `"deleted\\n"`)
argBytes, readErr := os.ReadFile(argFile.Name())
c.Assert(readErr, IsNil)
argText := string(argBytes)
c.Check(argText, Matches, `(?s).*--batch\n--yes\n.*`)
c.Check(argText, Matches, `(?s).*--keyring\n/trustedkeys\.gpg\n.*`)
c.Check(argText, Matches, `(?s).*--delete-keys\n8B48AD6246925553\n.*`)
})
}
// TestAPIGPGListKeysCommandFailure tests list error propagation from gpg
func (s *GPGSuite) TestAPIGPGListKeysCommandFailure(c *C) {
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
"echo 'keyring missing'\n" +
"exit 1\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, `(?s).*failed to list keys: keyring missing.*`)
})
}
// TestAPIGPGDeleteKeyCommandFailure tests delete error propagation from gpg
func (s *GPGSuite) TestAPIGPGDeleteKeyCommandFailure(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, "", "", "delete failed"), func(_ string) {
body, err := json.Marshal(gpgDeleteKeyParams{
Keyring: "trustedkeys.gpg",
GpgKeyID: "8B48AD6246925553",
})
c.Assert(err, IsNil)
response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, `(?s).*failed to delete key: delete failed.*`)
})
}
+12 -12
View File
@@ -67,17 +67,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
outC := make(chan string) outC := make(chan string)
go func() { go func() {
var buf bytes.Buffer var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader) io.Copy(&buf, s.logReader)
fmt.Println(buf.String()) fmt.Println(buf.String())
outC <- buf.String() outC <- buf.String()
}() }()
s.HTTPRequest(http.MethodGet, "/", nil) s.HTTPRequest(http.MethodGet, "/", nil)
_ = s.logWriter.Close() s.logWriter.Close()
capturedOutput := <-outC capturedOutput := <-outC
var jsonMap map[string]interface{} var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap) json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok { if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "warn") c.Check(val, Equals, "warn")
@@ -130,17 +130,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
outC := make(chan string) outC := make(chan string)
go func() { go func() {
var buf bytes.Buffer var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader) io.Copy(&buf, s.logReader)
fmt.Println(buf.String()) fmt.Println(buf.String())
outC <- buf.String() outC <- buf.String()
}() }()
s.HTTPRequest(http.MethodGet, "/api/healthy", nil) s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
_ = s.logWriter.Close() s.logWriter.Close()
capturedOutput := <-outC capturedOutput := <-outC
var jsonMap map[string]interface{} var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap) json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok { if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info") c.Check(val, Equals, "info")
@@ -153,17 +153,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
outC := make(chan string) outC := make(chan string)
go func() { go func() {
var buf bytes.Buffer var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader) io.Copy(&buf, s.logReader)
fmt.Println(buf.String()) fmt.Println(buf.String())
outC <- buf.String() outC <- buf.String()
}() }()
s.HTTPRequest(http.MethodGet, "/api/ready", nil) s.HTTPRequest(http.MethodGet, "/api/ready", nil)
_ = s.logWriter.Close() s.logWriter.Close()
capturedOutput := <-outC capturedOutput := <-outC
var jsonMap map[string]interface{} var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap) json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok { if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "error") c.Check(val, Equals, "error")
@@ -176,17 +176,17 @@ func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
outC := make(chan string) outC := make(chan string)
go func() { go func() {
var buf bytes.Buffer var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader) io.Copy(&buf, s.logReader)
fmt.Println(buf.String()) fmt.Println(buf.String())
outC <- buf.String() outC <- buf.String()
}() }()
s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil) s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
_ = s.logWriter.Close() s.logWriter.Close()
capturedOutput := <-outC capturedOutput := <-outC
var jsonMap map[string]interface{} var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap) json.Unmarshal([]byte(capturedOutput), &jsonMap)
fmt.Println(capturedOutput) fmt.Println(capturedOutput)
+58 -185
View File
@@ -31,61 +31,22 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
return verifier, nil return verifier, nil
} }
// stringSlicesEqual compares two string slices for equality (order matters)
func stringSlicesEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// uniqueStrings returns a new slice with only unique strings from the input, sorted
func uniqueStrings(input []string) []string {
if len(input) == 0 {
return input
}
seen := make(map[string]struct{}, len(input))
result := make([]string, 0, len(input))
for _, s := range input {
if _, ok := seen[s]; !ok {
seen[s] = struct{}{}
result = append(result, s)
}
}
sort.Strings(result)
return result
}
// @Summary List Mirrors // @Summary List Mirrors
// @Description **Show list of currently available mirrors** // @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API. // @Description Each mirror is returned as in “show” API.
// @Tags Mirrors // @Tags Mirrors
// @Produce json // @Produce json
// @Success 200 {array} remoteRepoResponse // @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get] // @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) { func apiMirrorsList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory() collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection() collection := collectionFactory.RemoteRepoCollection()
result := []remoteRepoResponse{} result := []*deb.RemoteRepo{}
err := collection.ForEach(func(repo *deb.RemoteRepo) error { collection.ForEach(func(repo *deb.RemoteRepo) error {
err := collection.LoadComplete(repo) result = append(result, repo)
if err != nil {
return err
}
result = append(result, newRemoteRepoResponse(repo))
return nil return nil
}) })
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
return
}
c.JSON(200, result) c.JSON(200, result)
} }
@@ -111,8 +72,6 @@ type mirrorCreateParams struct {
DownloadUdebs bool ` json:"DownloadUdebs"` DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files // Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"` DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to mirror AppStream (DEP-11) metadata
DownloadAppStream bool ` json:"DownloadAppStream"`
// Set "true" to include dependencies of matching packages when filtering // Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"` FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file // Set "true" to skip if the given components are in the Release file
@@ -164,7 +123,7 @@ func apiMirrorsCreate(c *gin.Context) {
} }
repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures, repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream) b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
if err != nil { if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err)) AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
@@ -273,7 +232,6 @@ func apiMirrorsShow(c *gin.Context) {
err = collection.LoadComplete(repo) err = collection.LoadComplete(repo)
if err != nil { if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err)) AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
return
} }
c.JSON(200, repo) c.JSON(200, repo)
@@ -361,7 +319,7 @@ func apiMirrorsPackages(c *gin.Context) {
} }
if c.Request.URL.Query().Get("format") == "details" { if c.Request.URL.Query().Get("format") == "details" {
_ = list.ForEach(func(p *deb.Package) error { list.ForEach(func(p *deb.Package) error {
result = append(result, p) result = append(result, p)
return nil return nil
}) })
@@ -372,133 +330,29 @@ func apiMirrorsPackages(c *gin.Context) {
} }
} }
type mirrorEditParams struct {
// Package query that is applied to mirror packages
Filter *string ` json:"Filter" example:"xserver-xorg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps *bool ` json:"FilterWithDeps"`
// Set "true" to mirror installer files
DownloadInstaller *bool `json:"DownloadInstaller"`
// Set "true" to mirror source packages
DownloadSources *bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs *bool ` json:"DownloadUdebs"`
// URL of the archive to mirror
ArchiveURL *string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Comma separated list of architectures
Architectures *[]string `json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file if a mirror update is required.
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures *bool ` json:"IgnoreSignatures"`
}
// @Summary Edit Mirror
// @Description **Edit mirror config**
// @Tags Mirrors
// @Param name path string true "mirror name to edit"
// @Consume json
// @Param request body mirrorEditParams true "Parameters"
// @Produce json
// @Success 200 {object} deb.RemoteRepo "Mirror was edited successfully"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 409 {object} Error "Aptly db locked"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [post]
func apiMirrorsEdit(c *gin.Context) {
var (
err error
b mirrorEditParams
repo *deb.RemoteRepo
)
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
name := c.Params.ByName("name")
repo, err = collection.ByName(name)
if err != nil {
AbortWithJSONError(c, 404, fmt.Errorf("unable to edit: %s", err))
return
}
err = repo.CheckLock()
if err != nil {
AbortWithJSONError(c, 409, fmt.Errorf("unable to edit: %s", err))
return
}
if c.Bind(&b) != nil {
return
}
fetchMirror := false
ignoreSignatures := context.Config().GpgDisableVerify
if b.Filter != nil {
repo.Filter = *b.Filter
}
if b.FilterWithDeps != nil {
repo.FilterWithDeps = *b.FilterWithDeps
}
if b.DownloadInstaller != nil {
repo.DownloadInstaller = *b.DownloadInstaller
}
if b.DownloadSources != nil {
repo.DownloadSources = *b.DownloadSources
}
if b.DownloadUdebs != nil {
repo.DownloadUdebs = *b.DownloadUdebs
}
if b.ArchiveURL != nil && *b.ArchiveURL != repo.ArchiveRoot {
repo.SetArchiveRoot(*b.ArchiveURL)
fetchMirror = true
}
if b.Architectures != nil {
uniqueArchitectures := uniqueStrings(*b.Architectures)
if !stringSlicesEqual(uniqueArchitectures, uniqueStrings(repo.Architectures)) {
repo.Architectures = uniqueArchitectures
fetchMirror = true
}
}
if b.IgnoreSignatures != nil {
ignoreSignatures = *b.IgnoreSignatures
}
if repo.IsFlat() && repo.DownloadUdebs {
AbortWithJSONError(c, 400, fmt.Errorf("unable to edit: flat mirrors don't support udebs"))
return
}
if fetchMirror {
verifier, err := getVerifier(b.Keyrings)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
return
}
}
err = collection.Update(repo)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
return
}
c.JSON(200, repo)
}
type mirrorUpdateParams struct { type mirrorUpdateParams struct {
// Change mirror name to `Name` // Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"` Name string ` json:"Name" example:"mirror1"`
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file // Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"` Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors // Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"` IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures // Set "true" to skip the verification of Release file signatures
@@ -507,8 +361,6 @@ type mirrorUpdateParams struct {
ForceUpdate bool ` json:"ForceUpdate"` ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages // Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"` SkipExistingPackages bool ` json:"SkipExistingPackages"`
// Set "true" to download only the latest version per package/architecture
LatestOnly bool ` json:"LatestOnly"`
} }
// @Summary Update Mirror // @Summary Update Mirror
@@ -542,6 +394,14 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
b.Name = remote.Name b.Name = remote.Name
b.DownloadUdebs = remote.DownloadUdebs
b.DownloadSources = remote.DownloadSources
b.SkipComponentCheck = remote.SkipComponentCheck
b.SkipArchitectureCheck = remote.SkipArchitectureCheck
b.FilterWithDeps = remote.FilterWithDeps
b.Filter = remote.Filter
b.Architectures = remote.Architectures
b.Components = remote.Components
b.IgnoreSignatures = context.Config().GpgDisableVerify b.IgnoreSignatures = context.Config().GpgDisableVerify
log.Info().Msgf("%s: Starting mirror update", b.Name) log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -558,6 +418,27 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
} }
if b.DownloadUdebs != remote.DownloadUdebs {
if remote.IsFlat() && b.DownloadUdebs {
AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
return
}
}
if b.ArchiveURL != "" {
remote.SetArchiveRoot(b.ArchiveURL)
}
remote.Name = b.Name
remote.DownloadUdebs = b.DownloadUdebs
remote.DownloadSources = b.DownloadSources
remote.SkipComponentCheck = b.SkipComponentCheck
remote.SkipArchitectureCheck = b.SkipArchitectureCheck
remote.FilterWithDeps = b.FilterWithDeps
remote.Filter = b.Filter
remote.Architectures = b.Architectures
remote.Components = b.Components
verifier, err := getVerifier(b.Keyrings) verifier, err := getVerifier(b.Keyrings)
if err != nil { if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err)) AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -580,19 +461,11 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
} }
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck) err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
} }
if remote.DownloadAppStream && !remote.IsFlat() {
err = remote.DownloadAppStreamFiles(out, downloader,
context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
}
if remote.Filter != "" { if remote.Filter != "" {
var filterQuery deb.PackageQuery var filterQuery deb.PackageQuery
@@ -608,7 +481,7 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly) collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
} }
@@ -618,7 +491,7 @@ func apiMirrorsUpdate(c *gin.Context) {
e := context.ReOpenDatabase() e := context.ReOpenDatabase()
if e == nil { if e == nil {
remote.MarkAsIdle() remote.MarkAsIdle()
_ = collection.Update(remote) collection.Update(remote)
} }
}() }()
@@ -706,7 +579,7 @@ func apiMirrorsUpdate(c *gin.Context) {
file, e = os.CreateTemp("", task.File.Filename) file, e = os.CreateTemp("", task.File.Filename)
if e == nil { if e == nil {
task.TempDownPath = file.Name() task.TempDownPath = file.Name()
_ = file.Close() file.Close()
} }
} }
if e != nil { if e != nil {
@@ -780,7 +653,7 @@ func apiMirrorsUpdate(c *gin.Context) {
} }
log.Info().Msgf("%s: Finalizing download...", b.Name) log.Info().Msgf("%s: Finalizing download...", b.Name)
_ = remote.FinalizeDownload(collectionFactory, out) remote.FinalizeDownload(collectionFactory, out)
err = collectionFactory.RemoteRepoCollection().Update(remote) err = collectionFactory.RemoteRepoCollection().Update(remote)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
+2 -67
View File
@@ -4,13 +4,12 @@ import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
) )
type MirrorSuite struct { type MirrorSuite struct {
APISuite ApiSuite
} }
var _ = Suite(&MirrorSuite{}) var _ = Suite(&MirrorSuite{})
@@ -18,10 +17,7 @@ var _ = Suite(&MirrorSuite{})
func (s *MirrorSuite) TestGetMirrors(c *C) { func (s *MirrorSuite) TestGetMirrors(c *C) {
response, _ := s.HTTPRequest("GET", "/api/mirrors", nil) response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Check(response.Code, Equals, 200) c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "[]")
var mirrors []map[string]interface{}
err := json.Unmarshal(response.Body.Bytes(), &mirrors)
c.Assert(err, IsNil)
} }
func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) { func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
@@ -30,21 +26,6 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}") c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
} }
func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
body, err := json.Marshal(gin.H{
"Name": "test-flat-appstream",
"ArchiveURL": "http://example.com/repo/",
"Distribution": "./",
"DownloadAppStream": true,
})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
}
func (s *MirrorSuite) TestCreateMirror(c *C) { func (s *MirrorSuite) TestCreateMirror(c *C) {
c.ExpectFailure("Need to mock downloads") c.ExpectFailure("Need to mock downloads")
body, err := json.Marshal(gin.H{ body, err := json.Marshal(gin.H{
@@ -57,49 +38,3 @@ func (s *MirrorSuite) TestCreateMirror(c *C) {
c.Check(response.Code, Equals, 400) c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Equals, "") c.Check(response.Body.String(), Equals, "")
} }
func (s *MirrorSuite) TestGetMirrorsIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().RemoteRepoCollection()
repo, err := deb.NewRemoteRepo("count-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
c.Assert(err, IsNil)
err = collection.Add(repo)
c.Assert(err, IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), makePackageRefList(c).Encode())
response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var mirrors []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &mirrors)
c.Assert(err, IsNil)
found := false
for _, mirror := range mirrors {
if mirror["Name"] == "count-mirror" {
found = true
value, ok := mirror["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *MirrorSuite) TestGetMirrorsReturns500OnCorruptRefList(c *C) {
collection := s.context.NewCollectionFactory().RemoteRepoCollection()
repo, err := deb.NewRemoteRepo("broken-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
c.Assert(err, IsNil)
c.Assert(collection.Add(repo), IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*unable to show:.*")
}
-30
View File
@@ -1,30 +0,0 @@
package api
import "github.com/aptly-dev/aptly/deb"
type remoteRepoResponse struct {
*deb.RemoteRepo
NumPackages int `json:"NumPackages"`
}
type localRepoResponse struct {
*deb.LocalRepo
NumPackages int `json:"NumPackages"`
}
type snapshotResponse struct {
*deb.Snapshot
NumPackages int `json:"NumPackages"`
}
func newRemoteRepoResponse(repo *deb.RemoteRepo) remoteRepoResponse {
return remoteRepoResponse{RemoteRepo: repo, NumPackages: repo.NumPackages()}
}
func newLocalRepoResponse(repo *deb.LocalRepo) localRepoResponse {
return localRepoResponse{LocalRepo: repo, NumPackages: repo.NumPackages()}
}
func newSnapshotResponse(snapshot *deb.Snapshot) snapshotResponse {
return snapshotResponse{Snapshot: snapshot, NumPackages: snapshot.NumPackages()}
}
-19
View File
@@ -1,19 +0,0 @@
package api
import (
"github.com/aptly-dev/aptly/deb"
. "gopkg.in/check.v1"
)
func makePackageRefList(c *C) *deb.PackageRefList {
list := deb.NewPackageList()
c.Assert(list.Add(&deb.Package{Name: "libcount", Version: "1.0", Architecture: "amd64"}), IsNil)
c.Assert(list.Add(&deb.Package{Name: "appcount", Version: "2.0", Architecture: "all"}), IsNil)
return deb.NewPackageRefListFromPackageList(list)
}
func putRawDBValue(c *C, s *APISuite, key []byte, value []byte) {
db, err := s.context.Database()
c.Assert(err, IsNil)
c.Assert(db.Put(key, value), IsNil)
}
+1 -1
View File
@@ -5,7 +5,7 @@ import (
) )
type PackagesSuite struct { type PackagesSuite struct {
APISuite ApiSuite
} }
var _ = Suite(&PackagesSuite{}) var _ = Suite(&PackagesSuite{})
+13 -86
View File
@@ -16,8 +16,8 @@ import (
type signingParams struct { type signingParams struct {
// Don't sign published repository // Don't sign published repository
Skip bool ` json:"Skip" example:"false"` Skip bool ` json:"Skip" example:"false"`
// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used // GPG key ID to use when signing the release, if not specified default key is used
GpgKey string ` json:"GpgKey" example:"KEY_ID_a, KEY_ID_b"` GpgKey string ` json:"GpgKey" example:"A0546A43624A8331"`
// GPG keyring to use (instead of default) // GPG keyring to use (instead of default)
Keyring string ` json:"Keyring" example:"trustedkeys.gpg"` Keyring string ` json:"Keyring" example:"trustedkeys.gpg"`
// GPG secret keyring to use (instead of default) Note: depreciated with gpg2 // GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -41,21 +41,7 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
} }
signer := context.GetSigner() signer := context.GetSigner()
signer.SetKey(options.GpgKey)
var multiGpgKeys []string
// REST params have priority over config
if options.GpgKey != "" {
for _, p := range strings.Split(options.GpgKey, ",") {
if t := strings.TrimSpace(p); t != "" {
multiGpgKeys = append(multiGpgKeys, t)
}
}
} else if len(context.Config().GpgKeys) > 0 {
multiGpgKeys = context.Config().GpgKeys
}
for _, gpgKey := range multiGpgKeys {
signer.SetKey(gpgKey)
}
signer.SetKeyRing(options.Keyring, options.SecretKeyring) signer.SetKeyRing(options.Keyring, options.SecretKeyring)
signer.SetPassphrase(options.Passphrase, options.PassphraseFile) signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
@@ -182,12 +168,8 @@ type publishedRepoCreateParams struct {
SkipBz2 *bool ` json:"SkipBz2" example:"false"` SkipBz2 *bool ` json:"SkipBz2" example:"false"`
// Provide index files by hash // Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"` AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file.
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions // Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"` MultiDist *bool ` json:"MultiDist" example:"false"`
// Version of the release
Version string ` json:"Version" example:""`
} }
// @Summary Create Published Repository // @Summary Create Published Repository
@@ -359,17 +341,9 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
published.AcquireByHash = *b.AcquireByHash published.AcquireByHash = *b.AcquireByHash
} }
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.Version != "" {
published.Version = b.Version
}
duplicate := collection.CheckDuplicate(published) duplicate := collection.CheckDuplicate(published)
if duplicate != nil { if duplicate != nil {
_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
} }
@@ -402,16 +376,8 @@ type publishedRepoUpdateSwitchParams struct {
Snapshots []sourceParams ` json:"Snapshots"` Snapshots []sourceParams ` json:"Snapshots"`
// Provide index files by hash // Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"` AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions // Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"` MultiDist *bool ` json:"MultiDist" example:"false"`
// Value of Label: field in published repository stanza
Label *string ` json:"Label" example:"Debian"`
// Value of Origin: field in published repository stanza
Origin *string ` json:"Origin" example:"Debian"`
// Version of the release: Optional
Version *string ` json:"Version" example:"13.3"`
} }
// @Summary Update Published Repository // @Summary Update Published Repository
@@ -427,6 +393,7 @@ type publishedRepoUpdateSwitchParams struct {
// @Description // @Description
// @Description See also: `aptly publish update` / `aptly publish switch` // @Description See also: `aptly publish update` / `aptly publish switch`
// @Tags Publish // @Tags Publish
// @Produce json
// @Param prefix path string true "publishing prefix" // @Param prefix path string true "publishing prefix"
// @Param distribution path string true "distribution name" // @Param distribution path string true "distribution name"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
@@ -495,32 +462,16 @@ func apiPublishUpdateSwitch(c *gin.Context) {
published.AcquireByHash = *b.AcquireByHash published.AcquireByHash = *b.AcquireByHash
} }
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.MultiDist != nil { if b.MultiDist != nil {
published.MultiDist = *b.MultiDist published.MultiDist = *b.MultiDist
} }
if b.Label != nil {
published.Label = *b.Label
}
if b.Origin != nil {
published.Origin = *b.Origin
}
if b.Version != nil {
published.Version = *b.Version
}
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.LoadComplete(published, collectionFactory) err = collection.LoadComplete(published, collectionFactory)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
} }
revision := published.ObtainRevision() revision := published.ObtainRevision()
@@ -536,12 +487,12 @@ func apiPublishUpdateSwitch(c *gin.Context) {
result, err := published.Update(collectionFactory, out) result, err := published.Update(collectionFactory, out)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
} }
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
} }
err = collection.Update(published) err = collection.Update(published)
@@ -674,7 +625,7 @@ func apiPublishAddSource(c *gin.Context) {
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published) err = collection.Update(published)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -788,7 +739,7 @@ func apiPublishSetSources(c *gin.Context) {
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published) err = collection.Update(published)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -838,7 +789,7 @@ func apiPublishDropChanges(c *gin.Context) {
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published) err = collection.Update(published)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -918,7 +869,7 @@ func apiPublishUpdateSource(c *gin.Context) {
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published) err = collection.Update(published)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -981,7 +932,7 @@ func apiPublishRemoveSource(c *gin.Context) {
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published) err = collection.Update(published)
if err != nil { if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
@@ -1004,16 +955,8 @@ type publishedRepoUpdateParams struct {
SkipCleanup *bool ` json:"SkipCleanup" example:"false"` SkipCleanup *bool ` json:"SkipCleanup" example:"false"`
// Provide index files by hash // Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"` AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions // Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"` MultiDist *bool ` json:"MultiDist" example:"false"`
// Value of Label: field in published repository stanza
Label *string ` json:"Label" example:"Debian"`
// Value of Origin: field in published repository stanza
Origin *string ` json:"Origin" example:"Debian"`
// Version of the release: Optional
Version *string ` json:"Version" example:"13.3"`
} }
// @Summary Update Published Repository // @Summary Update Published Repository
@@ -1078,26 +1021,10 @@ func apiPublishUpdate(c *gin.Context) {
published.AcquireByHash = *b.AcquireByHash published.AcquireByHash = *b.AcquireByHash
} }
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.MultiDist != nil { if b.MultiDist != nil {
published.MultiDist = *b.MultiDist published.MultiDist = *b.MultiDist
} }
if b.Label != nil {
published.Label = *b.Label
}
if b.Origin != nil {
published.Origin = *b.Origin
}
if b.Version != nil {
published.Version = *b.Version
}
resources := []string{string(published.Key())} resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
+42 -59
View File
@@ -24,19 +24,19 @@ import (
// @Tags Repos // @Tags Repos
// @Produce html // @Produce html
// @Success 200 {object} string "HTML" // @Success 200 {object} string "HTML"
// @Router /repos [get] // @Router /api/repos [get]
func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc { func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
c.Writer.Flush() c.Writer.Flush()
_, _ = c.Writer.WriteString("<pre>\n") c.Writer.WriteString("<pre>\n")
if len(localRepos) == 0 { if len(localRepos) == 0 {
_, _ = c.Writer.WriteString("<a href=\"-/\">default</a>\n") c.Writer.WriteString("<a href=\"-/\">default</a>\n")
} }
for publishPrefix := range localRepos { for publishPrefix := range localRepos {
_, _ = c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix)) c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
} }
_, _ = c.Writer.WriteString("</pre>") c.Writer.WriteString("</pre>")
c.Writer.Flush() c.Writer.Flush()
} }
} }
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
// @Param pkgPath path string true "Package Path" allowReserved=true // @Param pkgPath path string true "Package Path" allowReserved=true
// @Produce json // @Produce json
// @Success 200 "" // @Success 200 ""
// @Router /repos/{storage}/{pkgPath} [get] // @Router /api/{storage}/{pkgPath} [get]
func reposServeInAPIMode(c *gin.Context) { func reposServeInAPIMode(c *gin.Context) {
pkgpath := c.Param("pkgPath") pkgpath := c.Param("pkgPath")
@@ -69,26 +69,17 @@ func reposServeInAPIMode(c *gin.Context) {
// @Description Each repo is returned as in “show” API. // @Description Each repo is returned as in “show” API.
// @Tags Repos // @Tags Repos
// @Produce json // @Produce json
// @Success 200 {array} localRepoResponse // @Success 200 {array} deb.LocalRepo
// @Router /api/repos [get] // @Router /api/repos [get]
func apiReposList(c *gin.Context) { func apiReposList(c *gin.Context) {
result := []localRepoResponse{} result := []*deb.LocalRepo{}
collectionFactory := context.NewCollectionFactory() collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection() collection := collectionFactory.LocalRepoCollection()
err := collection.ForEach(func(r *deb.LocalRepo) error { collection.ForEach(func(r *deb.LocalRepo) error {
err := collection.LoadComplete(r) result = append(result, r)
if err != nil {
return err
}
result = append(result, newLocalRepoResponse(r))
return nil return nil
}) })
if err != nil {
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, result) c.JSON(200, result)
} }
@@ -116,9 +107,9 @@ type repoCreateParams struct {
// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""} // @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
// @Description ``` // @Description ```
// @Tags Repos // @Tags Repos
// @Produce json
// @Consume json // @Consume json
// @Param request body repoCreateParams true "Parameters" // @Param request body repoCreateParams true "Parameters"
// @Produce json
// @Success 201 {object} deb.LocalRepo // @Success 201 {object} deb.LocalRepo
// @Failure 404 {object} Error "Source snapshot not found" // @Failure 404 {object} Error "Source snapshot not found"
// @Failure 409 {object} Error "Local repo already exists" // @Failure 409 {object} Error "Local repo already exists"
@@ -187,10 +178,8 @@ type reposEditParams struct {
// @Summary Update Repository // @Summary Update Repository
// @Description **Update local repository meta information** // @Description **Update local repository meta information**
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name"
// @Consume json
// @Param request body reposEditParams true "Parameters"
// @Produce json // @Produce json
// @Param request body reposEditParams true "Parameters"
// @Success 200 {object} deb.LocalRepo "msg" // @Success 200 {object} deb.LocalRepo "msg"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error" // @Failure 500 {object} Error "Internal Server Error"
@@ -204,18 +193,17 @@ func apiReposEdit(c *gin.Context) {
collectionFactory := context.NewCollectionFactory() collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection() collection := collectionFactory.LocalRepoCollection()
name := c.Params.ByName("name") repo, err := collection.ByName(c.Params.ByName("name"))
repo, err := collection.ByName(name)
if err != nil { if err != nil {
AbortWithJSONError(c, 404, err) AbortWithJSONError(c, 404, err)
return return
} }
if b.Name != nil && *b.Name != name { if b.Name != nil {
_, err := collection.ByName(*b.Name) _, err := collection.ByName(*b.Name)
if err == nil { if err == nil {
// already exists // already exists
AbortWithJSONError(c, 404, fmt.Errorf("local repo with name %q already exists", *b.Name)) AbortWithJSONError(c, 404, err)
return return
} }
repo.Name = *b.Name repo.Name = *b.Name
@@ -243,8 +231,8 @@ func apiReposEdit(c *gin.Context) {
// @Summary Get Repository Info // @Summary Get Repository Info
// @Description Returns basic information about local repository. // @Description Returns basic information about local repository.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name"
// @Produce json // @Produce json
// @Param name path string true "Repository name"
// @Success 200 {object} deb.LocalRepo // @Success 200 {object} deb.LocalRepo
// @Failure 404 {object} Error "Repository not found" // @Failure 404 {object} Error "Repository not found"
// @Router /api/repos/{name} [get] // @Router /api/repos/{name} [get]
@@ -266,10 +254,9 @@ func apiReposShow(c *gin.Context) {
// @Description Cannot drop repos that are published. // @Description Cannot drop repos that are published.
// @Description Needs force=1 to drop repos used as source by other repos. // @Description Needs force=1 to drop repos used as source by other repos.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Produce json
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Param force query int false "force: 1 to enable" // @Param force query int false "force: 1 to enable"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "Repo object" // @Success 200 {object} task.ProcessReturnValue "Repo object"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Repo Conflict" // @Failure 404 {object} Error "Repo Conflict"
@@ -319,12 +306,12 @@ func apiReposDrop(c *gin.Context) {
// @Description ["Pi386 aptly 0.8 966561016b44ed80"] // @Description ["Pi386 aptly 0.8 966561016b44ed80"]
// @Description ``` // @Description ```
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Produce json
// @Param name path string true "Snapshot to search"
// @Param q query string true "Package query (e.g Name%20(~%20matlab))" // @Param q query string true "Package query (e.g Name%20(~%20matlab))"
// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query" // @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
// @Param format query string true "Set to 'details' to return extra info about each package" // @Param format query string true "Set to 'details' to return extra info about each package"
// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name" // @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
// @Produce json
// @Success 200 {object} string "msg" // @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Internal Server Error" // @Failure 404 {object} Error "Internal Server Error"
@@ -419,11 +406,9 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
// @Description // @Description
// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages cant be part of the same local repository. // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages cant be part of the same local repository.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Produce json
// @Consume json
// @Param request body reposPackagesAddDeleteParams true "Parameters" // @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} string "msg" // @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request" // @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
@@ -441,11 +426,9 @@ func apiReposPackagesAdd(c *gin.Context) {
// @Description // @Description
// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages. // @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name"
// @Param _async query bool false "Run in background and return task object"
// @Consume json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Produce json // @Produce json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "msg" // @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request" // @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
@@ -466,7 +449,7 @@ func apiReposPackagesDelete(c *gin.Context) {
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Param name path string true "Repository name"
// @Param dir path string true "Directory of packages" // @Param dir path string true "Directory of packages"
// @Param file path string true "Filename" // @Param file path string false "Filename (optional)"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json // @Produce json
// @Success 200 {string} string "OK" // @Success 200 {string} string "OK"
@@ -587,7 +570,7 @@ func apiReposPackageFromDir(c *gin.Context) {
} }
// atempt to remove dir, if it fails, that's fine: probably it's not empty // atempt to remove dir, if it fails, that's fine: probably it's not empty
_ = os.Remove(filepath.Join(context.UploadPath(), dirParam)) os.Remove(filepath.Join(context.UploadPath(), dirParam))
} }
if failedFiles == nil { if failedFiles == nil {
@@ -624,11 +607,11 @@ type reposCopyPackageParams struct {
// @Summary Copy Package // @Summary Copy Package
// @Description Copies a package from a source to destination repository // @Description Copies a package from a source to destination repository
// @Tags Repos // @Tags Repos
// @Param name path string true "Destination repo" // @Produce json
// @Param src path string true "Source repo" // @Param name path string true "Source repo"
// @Param src path string true "Destination repo"
// @Param file path string true "File/packages to copy" // @Param file path string true "File/packages to copy"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "msg" // @Success 200 {object} task.ProcessReturnValue "msg"
// @Failure 400 {object} Error "Bad Request" // @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
@@ -779,15 +762,12 @@ func apiReposCopyPackage(c *gin.Context) {
// @Summary Include File from Directory // @Summary Include File from Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Produce json
// @Param dir path string true "Directory of packages"
// @Param file path string true "File/packages to include"
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository" // @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} string "msg" // @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir}/{file} [post] // @Router /api/repos/{name}/include/{dir}/{file} [post]
@@ -796,22 +776,26 @@ func apiReposIncludePackageFromFile(c *gin.Context) {
apiReposIncludePackageFromDir(c) apiReposIncludePackageFromDir(c)
} }
type reposIncludePackageFromDirReport struct {
Warnings []string
Added []string
Deleted []string
}
type reposIncludePackageFromDirResponse struct { type reposIncludePackageFromDirResponse struct {
Report *aptly.RecordingResultReporter Report reposIncludePackageFromDirReport
FailedFiles []string FailedFiles []string
} }
// @Summary Include Directory // @Summary Include Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api. // @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos // @Tags Repos
// @Param name path string true "Repository name" // @Produce json
// @Param dir path string true "Directory of packages"
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package" // @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository" // @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files" // @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature" // @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} reposIncludePackageFromDirResponse "Response" // @Success 200 {object} reposIncludePackageFromDirResponse "Response"
// @Failure 404 {object} Error "Not Found" // @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir} [post] // @Router /api/repos/{name}/include/{dir} [post]
@@ -852,7 +836,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
} }
var resources []string var resources []string
if len(repoTemplate.Root.Nodes) > 1 { if len(repoTemplate.Tree.Root.Nodes) > 1 {
resources = append(resources, task.AllLocalReposResourcesKey) resources = append(resources, task.AllLocalReposResourcesKey)
} else { } else {
// repo template string is simple text so only use resource key of specific repository // repo template string is simple text so only use resource key of specific repository
@@ -892,7 +876,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
if !noRemoveFiles { if !noRemoveFiles {
// atempt to remove dir, if it fails, that's fine: probably it's not empty // atempt to remove dir, if it fails, that's fine: probably it's not empty
_ = os.Remove(filepath.Join(context.UploadPath(), dirParam)) os.Remove(filepath.Join(context.UploadPath(), dirParam))
} }
if failedFiles == nil { if failedFiles == nil {
@@ -912,10 +896,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", ")) out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
} }
ret := reposIncludePackageFromDirResponse{ return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
Report: reporter, "Report": reporter,
FailedFiles: failedFiles, "FailedFiles": failedFiles,
} }}, nil
return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
}) })
} }
-63
View File
@@ -1,63 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
. "gopkg.in/check.v1"
)
type ReposSuite struct {
APISuite
}
var _ = Suite(&ReposSuite{})
func (s *ReposSuite) TestGetReposIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().LocalRepoCollection()
repo := deb.NewLocalRepo("count-repo-list", "")
repo.UpdateRefList(makePackageRefList(c))
c.Assert(collection.Add(repo), IsNil)
response, err := s.HTTPRequest("GET", "/api/repos", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var repos []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &repos)
c.Assert(err, IsNil)
found := false
for _, repo := range repos {
if repo["Name"] == "count-repo-list" {
found = true
value, ok := repo["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *ReposSuite) TestGetReposReturns500OnCorruptRefList(c *C) {
body, err := json.Marshal(gin.H{"Name": "broken-repo-list"})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 201)
collection := s.context.NewCollectionFactory().LocalRepoCollection()
repo, err := collection.ByName("broken-repo-list")
c.Assert(err, IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
response, err = s.HTTPRequest("GET", "/api/repos", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
}
+17 -21
View File
@@ -2,6 +2,7 @@ package api
import ( import (
"net/http" "net/http"
"os"
"sync/atomic" "sync/atomic"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
@@ -11,19 +12,13 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
// _ "github.com/aptly-dev/aptly/docs" // import docs "github.com/aptly-dev/aptly/docs"
// swaggerFiles "github.com/swaggo/files" swaggerFiles "github.com/swaggo/files"
// ginSwagger "github.com/swaggo/gin-swagger" ginSwagger "github.com/swaggo/gin-swagger"
) )
var context *ctx.AptlyContext var context *ctx.AptlyContext
// @Summary Get Metrics
// @Description **Get Prometheus Metrics**
// @Tags Status
// @Produce text/plain
// @Success 200 {string} string Metrics
// @Router /api/metrics [get]
func apiMetricsGet() gin.HandlerFunc { func apiMetricsGet() gin.HandlerFunc {
return func(c *gin.Context) { return func(c *gin.Context) {
countPackagesByRepos() countPackagesByRepos()
@@ -61,22 +56,26 @@ func Router(c *ctx.AptlyContext) http.Handler {
router.UseRawPath = true router.UseRawPath = true
if c.Config().LogFormat == "json" { if c.Config().LogFormat == "json" {
c.StructuredLogging(true)
utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
gin.DefaultWriter = utils.LogWriter{Logger: log.Logger} gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
router.Use(JSONLogger()) router.Use(JSONLogger())
} else { } else {
c.StructuredLogging(false)
utils.SetupDefaultLogger(c.Config().LogLevel)
router.Use(gin.Logger()) router.Use(gin.Logger())
} }
router.Use(gin.Recovery(), gin.ErrorLogger()) router.Use(gin.Recovery(), gin.ErrorLogger())
// if c.Config().EnableSwaggerEndpoint { if c.Config().EnableSwaggerEndpoint {
// router.GET("docs.html", func(c *gin.Context) { router.GET("docs.html", func(c *gin.Context) {
// c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML) c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML)
// }) })
// router.Use(redirectSwagger) router.Use(redirectSwagger)
// url := ginSwagger.URL("/docs/doc.json") url := ginSwagger.URL("/docs/doc.json")
// router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url)) router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
// } }
if c.Config().EnableMetricsEndpoint { if c.Config().EnableMetricsEndpoint {
MetricsCollectorRegistrar.Register(router) MetricsCollectorRegistrar.Register(router)
@@ -164,15 +163,12 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/mirrors/:name", apiMirrorsShow) api.GET("/mirrors/:name", apiMirrorsShow)
api.GET("/mirrors/:name/packages", apiMirrorsPackages) api.GET("/mirrors/:name/packages", apiMirrorsPackages)
api.POST("/mirrors", apiMirrorsCreate) api.POST("/mirrors", apiMirrorsCreate)
api.POST("/mirrors/:name", apiMirrorsEdit)
api.PUT("/mirrors/:name", apiMirrorsUpdate) api.PUT("/mirrors/:name", apiMirrorsUpdate)
api.DELETE("/mirrors/:name", apiMirrorsDrop) api.DELETE("/mirrors/:name", apiMirrorsDrop)
} }
{ {
api.GET("/gpg/keys", apiGPGListKeys)
api.POST("/gpg/key", apiGPGAddKey) api.POST("/gpg/key", apiGPGAddKey)
api.DELETE("/gpg/key", apiGPGDeleteKey)
} }
{ {
@@ -224,7 +220,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/graph.:ext", apiGraph) api.GET("/graph.:ext", apiGraph)
} }
{ {
api.POST("/db/cleanup", apiDBCleanup) api.POST("/db/cleanup", apiDbCleanup)
} }
{ {
api.GET("/tasks", apiTasksList) api.GET("/tasks", apiTasksList)
+9 -17
View File
@@ -20,7 +20,7 @@ import (
// @Description Each snapshot is returned as in “show” API. // @Description Each snapshot is returned as in “show” API.
// @Tags Snapshots // @Tags Snapshots
// @Produce json // @Produce json
// @Success 200 {array} snapshotResponse // @Success 200 {array} deb.Snapshot
// @Router /api/snapshots [get] // @Router /api/snapshots [get]
func apiSnapshotsList(c *gin.Context) { func apiSnapshotsList(c *gin.Context) {
SortMethodString := c.Request.URL.Query().Get("sort") SortMethodString := c.Request.URL.Query().Get("sort")
@@ -32,20 +32,11 @@ func apiSnapshotsList(c *gin.Context) {
SortMethodString = "name" SortMethodString = "name"
} }
result := []snapshotResponse{} result := []*deb.Snapshot{}
err := collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error { collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
err := collection.LoadComplete(snapshot) result = append(result, snapshot)
if err != nil {
return err
}
result = append(result, newSnapshotResponse(snapshot))
return nil return nil
}) })
if err != nil {
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, result) c.JSON(200, result)
} }
@@ -226,9 +217,10 @@ type snapshotsCreateFromRepositoryParams struct {
// @Summary Snapshot Repository // @Summary Snapshot Repository
// @Description **Create a snapshot of a repository by name** // @Description **Create a snapshot of a repository by name**
// @Tags Snapshots // @Tags Snapshots
// @Param name path string true "Repository name"
// @Consume json // @Consume json
// @Param request body snapshotsCreateFromRepositoryParams true "Parameters" // @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
// @Param name path string true "Repository name" // @Param name path string true "Name of the snapshot"
// @Param _async query bool false "Run in background and return task object" // @Param _async query bool false "Run in background and return task object"
// @Produce json // @Produce json
// @Success 201 {object} deb.Snapshot "Created snapshot object" // @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -563,7 +555,7 @@ func apiSnapshotsMerge(c *gin.Context) {
} }
if len(body.Sources) < 1 { if len(body.Sources) < 1 {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("minimum one source snapshot is required")) AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
return return
} }
@@ -773,7 +765,7 @@ func apiSnapshotsPull(c *gin.Context) {
addedPackages := []string{} addedPackages := []string{}
alreadySeen := map[string]bool{} alreadySeen := map[string]bool{}
_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error { destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key] _, seen := alreadySeen[key]
@@ -789,7 +781,7 @@ func apiSnapshotsPull(c *gin.Context) {
// If !allMatches, add only first matching name-arch package // If !allMatches, add only first matching name-arch package
if !seen || allMatches { if !seen || allMatches {
_ = toPackageList.Add(pkg) toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String()) addedPackages = append(addedPackages, pkg.String())
} }
-53
View File
@@ -1,53 +0,0 @@
package api
import (
"encoding/json"
"github.com/aptly-dev/aptly/deb"
. "gopkg.in/check.v1"
)
type SnapshotsSuite struct {
APISuite
}
var _ = Suite(&SnapshotsSuite{})
func (s *SnapshotsSuite) TestGetSnapshotsIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().SnapshotCollection()
snapshot := deb.NewSnapshotFromRefList("count-snapshot-list", nil, makePackageRefList(c), "")
c.Assert(collection.Add(snapshot), IsNil)
response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var snapshots []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &snapshots)
c.Assert(err, IsNil)
found := false
for _, snapshot := range snapshots {
if snapshot["Name"] == "count-snapshot-list" {
found = true
value, ok := snapshot["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *SnapshotsSuite) TestGetSnapshotsReturns500OnCorruptRefList(c *C) {
collection := s.context.NewCollectionFactory().SnapshotCollection()
snapshot := deb.NewSnapshotFromRefList("broken-snapshot-list", nil, makePackageRefList(c), "")
c.Assert(collection.Add(snapshot), IsNil)
putRawDBValue(c, &s.APISuite, snapshot.RefKey(), []byte("not-msgpack"))
response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
}
+1 -1
View File
@@ -1,4 +1,4 @@
package aptly package aptly
// AptlyConf holds the default aptly.conf (filled in at link time) // Default aptly.conf (filled in at link time)
var AptlyConf []byte var AptlyConf []byte
+46 -39
View File
@@ -5,28 +5,35 @@ package azure
import ( import (
"context" "context"
"encoding/hex" "encoding/hex"
"errors"
"fmt" "fmt"
"io" "io"
"net/url" "os"
"path/filepath" "path/filepath"
"time" "time"
"github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
) )
func isBlobNotFound(err error) bool { func isBlobNotFound(err error) bool {
storageError, ok := err.(azblob.StorageError) var respErr *azcore.ResponseError
return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound if errors.As(err, &respErr) {
return respErr.StatusCode == 404 // BlobNotFound
}
return false
} }
type azContext struct { type azContext struct {
container azblob.ContainerURL client *azblob.Client
container string
prefix string prefix string
} }
func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) { func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) cred, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -35,15 +42,14 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) (
endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName) endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
} }
url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container)) serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
result := &azContext{ result := &azContext{
container: containerURL, client: serviceClient,
container: container,
prefix: prefix, prefix: prefix,
} }
@@ -54,10 +60,6 @@ func (az *azContext) blobPath(path string) string {
return filepath.Join(az.prefix, path) return filepath.Join(az.prefix, path)
} }
func (az *azContext) blobURL(path string) azblob.BlobURL {
return az.container.NewBlobURL(az.blobPath(path))
}
func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) { func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
const delimiter = "/" const delimiter = "/"
paths = make([]string, 0, 1024) paths = make([]string, 0, 1024)
@@ -67,27 +69,33 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
prefix += delimiter prefix += delimiter
} }
for marker := (azblob.Marker{}); marker.NotDone(); { ctx := context.Background()
listBlob, err := az.container.ListBlobsFlatSegment( maxResults := int32(1)
context.Background(), marker, azblob.ListBlobsSegmentOptions{ pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{
Prefix: prefix, Prefix: &prefix,
MaxResults: 1, MaxResults: &maxResults,
Details: azblob.BlobListingDetails{Metadata: true}}) Include: azblob.ListBlobsInclude{Metadata: true},
})
// Iterate over each page
for pager.More() {
page, err := pager.NextPage(ctx)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err) return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
} }
marker = listBlob.NextMarker for _, blob := range page.Segment.BlobItems {
for _, blob := range listBlob.Segment.BlobItems {
if prefix == "" { if prefix == "" {
paths = append(paths, blob.Name) paths = append(paths, *blob.Name)
} else { } else {
paths = append(paths, blob.Name[len(prefix):]) name := *blob.Name
paths = append(paths, name[len(prefix):])
} }
md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5)) b := *blob
} md5 := b.Properties.ContentMD5
md5s = append(md5s, fmt.Sprintf("%x", md5))
}
if progress != nil { if progress != nil {
time.Sleep(time.Duration(500) * time.Millisecond) time.Sleep(time.Duration(500) * time.Millisecond)
progress.AddBar(1) progress.AddBar(1)
@@ -97,28 +105,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p
return paths, md5s, nil return paths, md5s, nil
} }
func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error { func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error {
uploadOptions := azblob.UploadStreamToBlockBlobOptions{ uploadOptions := &azblob.UploadFileOptions{
BufferSize: 4 * 1024 * 1024, BlockSize: 4 * 1024 * 1024,
MaxBuffers: 8, Concurrency: 8,
} }
path := az.blobPath(blobName)
if len(sourceMD5) > 0 { if len(sourceMD5) > 0 {
decodedMD5, err := hex.DecodeString(sourceMD5) decodedMD5, err := hex.DecodeString(sourceMD5)
if err != nil { if err != nil {
return err return err
} }
uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{ uploadOptions.HTTPHeaders = &blob.HTTPHeaders{
ContentMD5: decodedMD5, BlobContentMD5: decodedMD5,
} }
} }
_, err := azblob.UploadStreamToBlockBlob( var err error
context.Background(), if file, ok := source.(*os.File); ok {
source, _, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions)
blob.ToBlockBlobURL(), }
uploadOptions,
)
return err return err
} }
+21 -24
View File
@@ -5,7 +5,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils" "github.com/aptly-dev/aptly/utils"
"github.com/pkg/errors" "github.com/pkg/errors"
@@ -41,10 +40,7 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu
return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename) return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
} }
func (pool *PackagePool) ensureChecksums( func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) {
poolPath string,
checksumStorage aptly.ChecksumStorage,
) (*utils.ChecksumInfo, error) {
targetChecksums, err := checksumStorage.Get(poolPath) targetChecksums, err := checksumStorage.Get(poolPath)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -52,8 +48,7 @@ func (pool *PackagePool) ensureChecksums(
if targetChecksums == nil { if targetChecksums == nil {
// we don't have checksums stored yet for this file // we don't have checksums stored yet for this file
blob := pool.az.blobURL(poolPath) download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil)
download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
if err != nil { if err != nil {
if isBlobNotFound(err) { if isBlobNotFound(err) {
return nil, nil return nil, nil
@@ -63,7 +58,7 @@ func (pool *PackagePool) ensureChecksums(
} }
targetChecksums = &utils.ChecksumInfo{} targetChecksums = &utils.ChecksumInfo{}
*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{})) *targetChecksums, err = utils.ChecksumsForReader(download.Body)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath) return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
} }
@@ -92,46 +87,49 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er
} }
func (pool *PackagePool) Size(path string) (int64, error) { func (pool *PackagePool) Size(path string) (int64, error) {
blob := pool.az.blobURL(path) serviceClient := pool.az.client.ServiceClient()
props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) containerClient := serviceClient.NewContainerClient(pool.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.TODO(), nil)
if err != nil { if err != nil {
return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
} }
return props.ContentLength(), nil return *props.ContentLength, nil
} }
func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
blob := pool.az.blobURL(path)
temp, err := os.CreateTemp("", "blob-download") temp, err := os.CreateTemp("", "blob-download")
if err != nil { if err != nil {
return nil, errors.Wrap(err, "error creating temporary file for blob download") return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
} }
defer os.Remove(temp.Name()) defer os.Remove(temp.Name())
err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{}) _, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "error downloading blob at %s", path) return nil, errors.Wrapf(err, "error downloading blob %s", path)
} }
return temp, nil return temp, nil
} }
func (pool *PackagePool) Remove(path string) (int64, error) { func (pool *PackagePool) Remove(path string) (int64, error) {
blob := pool.az.blobURL(path) serviceClient := pool.az.client.ServiceClient()
props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) containerClient := serviceClient.NewContainerClient(pool.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.TODO(), nil)
if err != nil { if err != nil {
return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool) return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
} }
_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) _, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil)
if err != nil { if err != nil {
return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool) return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
} }
return props.ContentLength(), nil return *props.ContentLength, nil
} }
func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) { func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
@@ -145,7 +143,6 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
} }
path := pool.buildPoolPath(basename, checksums) path := pool.buildPoolPath(basename, checksums)
blob := pool.az.blobURL(path)
targetChecksums, err := pool.ensureChecksums(path, checksumStorage) targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
if err != nil { if err != nil {
return "", err return "", err
@@ -161,7 +158,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
} }
defer source.Close() defer source.Close()
err = pool.az.putFile(blob, source, checksums.MD5) err = pool.az.putFile(path, source, checksums.MD5)
if err != nil { if err != nil {
return "", err return "", err
} }
+5 -3
View File
@@ -7,7 +7,7 @@ import (
"path/filepath" "path/filepath"
"runtime" "runtime"
"github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils" "github.com/aptly-dev/aptly/utils"
@@ -50,8 +50,10 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint) s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil) c.Assert(err, IsNil)
cnt := s.pool.az.container publicAccessType := azblob.PublicAccessTypeContainer
_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil) c.Assert(err, IsNil)
s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint) s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
+67 -58
View File
@@ -3,20 +3,21 @@ package azure
import ( import (
"context" "context"
"fmt" "fmt"
"net/http"
"os" "os"
"path/filepath" "path/filepath"
"time" "time"
"github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils" "github.com/aptly-dev/aptly/utils"
"github.com/google/uuid"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
// PublishedStorage abstract file system with published files (actually hosted on Azure) // PublishedStorage abstract file system with published files (actually hosted on Azure)
type PublishedStorage struct { type PublishedStorage struct {
container azblob.ContainerURL
prefix string prefix string
az *azContext az *azContext
pathCache map[string]map[string]string pathCache map[string]map[string]string
@@ -66,7 +67,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
} }
defer source.Close() defer source.Close()
err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5) err = storage.az.putFile(path, source, sourceMD5)
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
} }
@@ -76,14 +77,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
// RemoveDirs removes directory structure under public path // RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error { func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
path = storage.az.blobPath(path)
filelist, err := storage.Filelist(path) filelist, err := storage.Filelist(path)
if err != nil { if err != nil {
return err return err
} }
for _, filename := range filelist { for _, filename := range filelist {
blob := storage.az.blobURL(filepath.Join(path, filename)) blob := filepath.Join(path, filename)
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil)
if err != nil { if err != nil {
return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err) return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
} }
@@ -94,8 +96,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error
// Remove removes single file under public path // Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error { func (storage *PublishedStorage) Remove(path string) error {
blob := storage.az.blobURL(path) path = storage.az.blobPath(path)
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil)
if err != nil { if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err)) err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
} }
@@ -114,9 +116,8 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
relFilePath := filepath.Join(publishedRelPath, fileName) relFilePath := filepath.Join(publishedRelPath, fileName)
// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
// FIXME: check how to integrate publishedPrefix: poolPath := storage.az.blobPath(prefixRelFilePath)
poolPath := storage.az.blobPath(fileName)
if storage.pathCache == nil { if storage.pathCache == nil {
storage.pathCache = make(map[string]map[string]string) storage.pathCache = make(map[string]map[string]string)
@@ -159,7 +160,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
} }
defer source.Close() defer source.Close()
err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5) err = storage.az.putFile(relFilePath, source, sourceMD5)
if err == nil { if err == nil {
pathCache[relFilePath] = sourceMD5 pathCache[relFilePath] = sourceMD5
} else { } else {
@@ -176,57 +177,58 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
} }
// Internal copy or move implementation // Internal copy or move implementation
func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error { func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error {
const leaseDuration = 30 const leaseDuration = 30
leaseID := uuid.NewString()
dstBlobURL := storage.az.blobURL(dst) serviceClient := storage.az.client.ServiceClient()
srcBlobURL := storage.az.blobURL(src) containerClient := serviceClient.NewContainerClient(storage.az.container)
leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{}) srcBlobClient := containerClient.NewBlobClient(src)
if err != nil || leaseResp.StatusCode() != http.StatusCreated { blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)})
return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL) if err != nil {
return fmt.Errorf("error acquiring lease on source blob %s", src)
} }
defer srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{})
srcBlobLeaseID := leaseResp.LeaseID()
copyResp, err := dstBlobURL.StartCopyFromURL( _, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil)
context.Background(), if err != nil {
srcBlobURL.URL(), return fmt.Errorf("error acquiring lease on source blob %s", src)
metadata, }
azblob.ModifiedAccessConditions{}, defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
azblob.BlobAccessConditions{},
azblob.DefaultAccessTier, dstBlobClient := containerClient.NewBlobClient(dst)
nil) copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
Metadata: metadata,
})
if err != nil { if err != nil {
return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err) return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err)
} }
copyStatus := copyResp.CopyStatus() copyStatus := *copyResp.CopyStatus
for { for {
if copyStatus == azblob.CopyStatusSuccess { if copyStatus == blob.CopyStatusTypeSuccess {
if move { if move {
_, err = srcBlobURL.Delete( _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{
context.Background(), AccessConditions: &blob.AccessConditions{
azblob.DeleteSnapshotsOptionNone, LeaseAccessConditions: &blob.LeaseAccessConditions{
azblob.BlobAccessConditions{ LeaseID: &leaseID,
LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}, },
}) },
})
return err return err
} }
return nil return nil
} else if copyStatus == azblob.CopyStatusPending { } else if copyStatus == blob.CopyStatusTypePending {
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
blobPropsResp, err := dstBlobURL.GetProperties( getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil)
context.Background(),
azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}},
azblob.ClientProvidedKeyOptions{})
if err != nil { if err != nil {
return fmt.Errorf("error getting destination blob properties %s", dstBlobURL) return fmt.Errorf("error getting copy progress %s", dst)
} }
copyStatus = blobPropsResp.CopyStatus() copyStatus = *getMetadata.CopyStatus
_, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{}) _, err = blobLeaseClient.RenewLease(context.Background(), nil)
if err != nil { if err != nil {
return fmt.Errorf("error renewing source blob lease %s", srcBlobURL) return fmt.Errorf("error renewing source blob lease %s", src)
} }
} else { } else {
return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus) return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus)
@@ -241,7 +243,9 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error {
// SymLink creates a copy of src file and adds link information as meta data // SymLink creates a copy of src file and adds link information as meta data
func (storage *PublishedStorage) SymLink(src string, dst string) error { func (storage *PublishedStorage) SymLink(src string, dst string) error {
return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */) metadata := make(map[string]*string)
metadata["SymLink"] = &src
return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */)
} }
// HardLink using symlink functionality as hard links do not exist // HardLink using symlink functionality as hard links do not exist
@@ -251,28 +255,33 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
// FileExists returns true if path exists // FileExists returns true if path exists
func (storage *PublishedStorage) FileExists(path string) (bool, error) { func (storage *PublishedStorage) FileExists(path string) (bool, error) {
blob := storage.az.blobURL(path) serviceClient := storage.az.client.ServiceClient()
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) containerClient := serviceClient.NewContainerClient(storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
if err != nil { if err != nil {
if isBlobNotFound(err) { if isBlobNotFound(err) {
return false, nil return false, nil
} }
return false, err return false, fmt.Errorf("error checking if blob %s exists: %v", path, err)
} else if resp.StatusCode() == http.StatusOK {
return true, nil
} }
return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) return true, nil
} }
// ReadLink returns the symbolic link pointed to by path. // ReadLink returns the symbolic link pointed to by path.
// This simply reads text file created with SymLink // This simply reads text file created with SymLink
func (storage *PublishedStorage) ReadLink(path string) (string, error) { func (storage *PublishedStorage) ReadLink(path string) (string, error) {
blob := storage.az.blobURL(path) serviceClient := storage.az.client.ServiceClient()
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) containerClient := serviceClient.NewContainerClient(storage.az.container)
blobClient := containerClient.NewBlobClient(path)
props, err := blobClient.GetProperties(context.Background(), nil)
if err != nil { if err != nil {
return "", err return "", fmt.Errorf("failed to get blob properties: %v", err)
} else if resp.StatusCode() != http.StatusOK {
return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode())
} }
return resp.NewMetadata()["SymLink"], nil
metadata := props.Metadata
if originalBlob, exists := metadata["original_blob"]; exists {
return *originalBlob, nil
}
return "", fmt.Errorf("error reading link %s: %v", path, err)
} }
+26 -23
View File
@@ -7,8 +7,11 @@ import (
"io/ioutil" "io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"bytes"
"github.com/Azure/azure-storage-blob-go/azblob" "github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils" "github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
@@ -66,8 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint) s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil) c.Assert(err, IsNil)
cnt := s.storage.az.container publicAccessType := azblob.PublicAccessTypeContainer
_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil) c.Assert(err, IsNil)
s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint) s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -75,41 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
} }
func (s *PublishedStorageSuite) TearDownTest(c *C) { func (s *PublishedStorageSuite) TearDownTest(c *C) {
cnt := s.storage.az.container _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
blob := s.storage.az.container.NewBlobURL(path) resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
c.Assert(err, IsNil) c.Assert(err, IsNil)
body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}) data, err := ioutil.ReadAll(resp.Body)
data, err := ioutil.ReadAll(body)
c.Assert(err, IsNil) c.Assert(err, IsNil)
return data return data
} }
func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
_, err := s.storage.az.container.NewBlobURL(path).GetProperties( serviceClient := s.storage.az.client.ServiceClient()
context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) containerClient := serviceClient.NewContainerClient(s.storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
c.Assert(err, NotNil) c.Assert(err, NotNil)
storageError, ok := err.(azblob.StorageError)
storageError, ok := err.(*azcore.ResponseError)
c.Assert(ok, Equals, true) c.Assert(ok, Equals, true)
c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound))) c.Assert(storageError.StatusCode, Equals, 404)
} }
func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
hash := md5.Sum(data) hash := md5.Sum(data)
_, err := azblob.UploadBufferToBlockBlob( uploadOptions := &azblob.UploadStreamOptions{
context.Background(), HTTPHeaders: &blob.HTTPHeaders{
data, BlobContentMD5: hash[:],
s.storage.az.container.NewBlockBlobURL(path), },
azblob.UploadToBlockBlobOptions{ }
BlobHTTPHeaders: azblob.BlobHTTPHeaders{ reader := bytes.NewReader(data)
ContentMD5: hash[:], _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
},
})
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
@@ -330,7 +333,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
// 2nd link from pool, providing wrong path for source file // 2nd link from pool, providing wrong path for source file
// //
// this test should check that file already exists in S3 and skip upload (which would fail if not skipped) // this test should check that file already exists in Azure and skip upload (which would fail if not skipped)
s.prefixedStorage.pathCache = nil s.prefixedStorage.pathCache = nil
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
c.Check(err, IsNil) c.Check(err, IsNil)
+4 -4
View File
@@ -46,7 +46,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
} }
if err == nil && len(listeners) == 1 { if err == nil && len(listeners) == 1 {
listener := listeners[0] listener := listeners[0]
defer func() { _ = listener.Close() }() defer listener.Close()
fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String()) fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
err = http.Serve(listener, api.Router(context)) err = http.Serve(listener, api.Router(context))
if err != nil { if err != nil {
@@ -67,7 +67,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
if _, ok := <-sigchan; ok { if _, ok := <-sigchan; ok {
fmt.Printf("\nShutdown signal received, waiting for background tasks...\n") fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
context.TaskList().Wait() context.TaskList().Wait()
_ = server.Shutdown(stdcontext.Background()) server.Shutdown(stdcontext.Background())
} }
})() })()
defer close(sigchan) defer close(sigchan)
@@ -75,14 +75,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
listenURL, err := url.Parse(listen) listenURL, err := url.Parse(listen)
if err == nil && listenURL.Scheme == "unix" { if err == nil && listenURL.Scheme == "unix" {
file := listenURL.Path file := listenURL.Path
_ = os.Remove(file) os.Remove(file)
var listener net.Listener var listener net.Listener
listener, err = net.Listen("unix", file) listener, err = net.Listen("unix", file)
if err != nil { if err != nil {
return fmt.Errorf("failed to listen on: %s\n%s", file, err) return fmt.Errorf("failed to listen on: %s\n%s", file, err)
} }
defer func() { _ = listener.Close() }() defer listener.Close()
err = server.Serve(listener) err = server.Serve(listener)
} else { } else {
+1 -1
View File
@@ -97,7 +97,7 @@ package environment to new version.`,
Flag: *flag.NewFlagSet("aptly", flag.ExitOnError), Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
Subcommands: []*commander.Command{ Subcommands: []*commander.Command{
makeCmdConfig(), makeCmdConfig(),
makeCmdDB(), makeCmdDb(),
makeCmdGraph(), makeCmdGraph(),
makeCmdMirror(), makeCmdMirror(),
makeCmdRepo(), makeCmdRepo(),
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"fmt" "fmt"
"github.com/smira/commander" "github.com/smira/commander"
yaml "gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
func aptlyConfigShow(_ *commander.Command, _ []string) error { func aptlyConfigShow(_ *commander.Command, _ []string) error {
+3 -3
View File
@@ -4,13 +4,13 @@ import (
"github.com/smira/commander" "github.com/smira/commander"
) )
func makeCmdDB() *commander.Command { func makeCmdDb() *commander.Command {
return &commander.Command{ return &commander.Command{
UsageLine: "db", UsageLine: "db",
Short: "manage aptly's internal database and package pool", Short: "manage aptly's internal database and package pool",
Subcommands: []*commander.Command{ Subcommands: []*commander.Command{
makeCmdDBCleanup(), makeCmdDbCleanup(),
makeCmdDBRecover(), makeCmdDbRecover(),
}, },
} }
} }
+7 -18
View File
@@ -12,7 +12,7 @@ import (
) )
// aptly db cleanup // aptly db cleanup
func aptlyDBCleanup(cmd *commander.Command, args []string) error { func aptlyDbCleanup(cmd *commander.Command, args []string) error {
var err error var err error
if len(args) != 0 { if len(args) != 0 {
@@ -26,7 +26,6 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
// collect information about references packages... // collect information about references packages...
existingPackageRefs := deb.NewPackageRefList() existingPackageRefs := deb.NewPackageRefList()
referencedAppStreamFiles := []string{}
// used only in verbose mode to report package use source // used only in verbose mode to report package use source
packageRefSources := map[string][]string{} packageRefSources := map[string][]string{}
@@ -49,17 +48,13 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose { if verbose {
description := fmt.Sprintf("mirror %s", repo.Name) description := fmt.Sprintf("mirror %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error { repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description) packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil return nil
}) })
} }
} }
for _, poolPath := range repo.AppStreamFiles {
referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
}
return nil return nil
}) })
if err != nil { if err != nil {
@@ -86,7 +81,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose { if verbose {
description := fmt.Sprintf("local repo %s", repo.Name) description := fmt.Sprintf("local repo %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error { repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description) packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil return nil
}) })
@@ -118,16 +113,11 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose { if verbose {
description := fmt.Sprintf("snapshot %s", snapshot.Name) description := fmt.Sprintf("snapshot %s", snapshot.Name)
_ = snapshot.RefList().ForEach(func(key []byte) error { snapshot.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description) packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil return nil
}) })
} }
for _, poolPath := range snapshot.AppStreamFiles {
referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
}
return nil return nil
}) })
if err != nil { if err != nil {
@@ -156,7 +146,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose { if verbose {
description := fmt.Sprintf("published repository %s:%s/%s component %s", description := fmt.Sprintf("published repository %s:%s/%s component %s",
published.Storage, published.Prefix, published.Distribution, component) published.Storage, published.Prefix, published.Distribution, component)
_ = published.RefList(component).ForEach(func(key []byte) error { published.RefList(component).ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description) packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil return nil
}) })
@@ -246,7 +236,6 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
return err return err
} }
referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
sort.Strings(referencedFiles) sort.Strings(referencedFiles)
context.Progress().ShutdownBar() context.Progress().ShutdownBar()
@@ -302,9 +291,9 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
return err return err
} }
func makeCmdDBCleanup() *commander.Command { func makeCmdDbCleanup() *commander.Command {
cmd := &commander.Command{ cmd := &commander.Command{
Run: aptlyDBCleanup, Run: aptlyDbCleanup,
UsageLine: "cleanup", UsageLine: "cleanup",
Short: "cleanup DB and package pool", Short: "cleanup DB and package pool",
Long: ` Long: `
+4 -45
View File
@@ -1,16 +1,13 @@
package cmd package cmd
import ( import (
"fmt"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander" "github.com/smira/commander"
"github.com/aptly-dev/aptly/database/goleveldb" "github.com/aptly-dev/aptly/database/goleveldb"
) )
// aptly db recover // aptly db recover
func aptlyDBRecover(cmd *commander.Command, args []string) error { func aptlyDbRecover(cmd *commander.Command, args []string) error {
var err error var err error
if len(args) != 0 { if len(args) != 0 {
@@ -19,19 +16,14 @@ func aptlyDBRecover(cmd *commander.Command, args []string) error {
} }
context.Progress().Printf("Recovering database...\n") context.Progress().Printf("Recovering database...\n")
if err = goleveldb.RecoverDB(context.DBPath()); err != nil { err = goleveldb.RecoverDB(context.DBPath())
return err
}
context.Progress().Printf("Checking database integrity...\n")
err = checkIntegrity()
return err return err
} }
func makeCmdDBRecover() *commander.Command { func makeCmdDbRecover() *commander.Command {
cmd := &commander.Command{ cmd := &commander.Command{
Run: aptlyDBRecover, Run: aptlyDbRecover,
UsageLine: "recover", UsageLine: "recover",
Short: "recover DB after crash", Short: "recover DB after crash",
Long: ` Long: `
@@ -46,36 +38,3 @@ Example:
return cmd return cmd
} }
func checkIntegrity() error {
return context.NewCollectionFactory().LocalRepoCollection().ForEach(checkRepo)
}
func checkRepo(repo *deb.LocalRepo) error {
collectionFactory := context.NewCollectionFactory()
repos := collectionFactory.LocalRepoCollection()
err := repos.LoadComplete(repo)
if err != nil {
return fmt.Errorf("load complete repo %q: %s", repo.Name, err)
}
dangling, err := deb.FindDanglingReferences(repo.RefList(), collectionFactory.PackageCollection())
if err != nil {
return fmt.Errorf("find dangling references: %w", err)
}
if len(dangling.Refs) > 0 {
for _, ref := range dangling.Refs {
context.Progress().Printf("Removing dangling database reference %q\n", ref)
}
repo.UpdateRefList(repo.RefList().Subtract(dangling))
if err = repos.Update(repo); err != nil {
return fmt.Errorf("update repo: %w", err)
}
}
return nil
}
+2 -2
View File
@@ -38,8 +38,8 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
if err != nil { if err != nil {
return err return err
} }
_ = tempfile.Close() tempfile.Close()
_ = os.Remove(tempfile.Name()) os.Remove(tempfile.Name())
format := context.Flags().Lookup("format").Value.String() format := context.Flags().Lookup("format").Value.String()
output := context.Flags().Lookup("output").Value.String() output := context.Flags().Lookup("output").Value.String()
+1 -1
View File
@@ -20,7 +20,7 @@ func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
verifier.AddKeyring(keyRing) verifier.AddKeyring(keyRing)
} }
err := verifier.InitKeyring(!ignoreSignatures) // be verbose only if verifying signatures is requested err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
if err != nil { if err != nil {
return nil, err return nil, err
} }
+1 -3
View File
@@ -20,7 +20,6 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources") downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool) downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool) downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") { if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool) ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
@@ -42,7 +41,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
} }
repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(), repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
downloadSources, downloadUdebs, downloadInstaller, downloadAppStream) downloadSources, downloadUdebs, downloadInstaller)
if err != nil { if err != nil {
return fmt.Errorf("unable to create mirror: %s", err) return fmt.Errorf("unable to create mirror: %s", err)
} }
@@ -101,7 +100,6 @@ Example:
} }
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-7
View File
@@ -35,8 +35,6 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
repo.Filter = flag.Value.String() // allows file/stdin with @ repo.Filter = flag.Value.String() // allows file/stdin with @
case "filter-with-deps": case "filter-with-deps":
repo.FilterWithDeps = flag.Value.Get().(bool) repo.FilterWithDeps = flag.Value.Get().(bool)
case "with-appstream":
repo.DownloadAppStream = flag.Value.Get().(bool)
case "with-installer": case "with-installer":
repo.DownloadInstaller = flag.Value.Get().(bool) repo.DownloadInstaller = flag.Value.Get().(bool)
case "with-sources": case "with-sources":
@@ -55,10 +53,6 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to edit: flat mirrors don't support udebs") return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
} }
if repo.IsFlat() && repo.DownloadAppStream {
return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
}
if repo.Filter != "" { if repo.Filter != "" {
_, err = query.Parse(repo.Filter) _, err = query.Parse(repo.Filter)
if err != nil { if err != nil {
@@ -113,7 +107,6 @@ Example:
AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin") AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well") cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files") cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages") cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)") cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+4 -4
View File
@@ -32,7 +32,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
repos := make([]string, collectionFactory.RemoteRepoCollection().Len()) repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
i := 0 i := 0
_ = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
if raw { if raw {
repos[i] = repo.Name repos[i] = repo.Name
} else { } else {
@@ -42,7 +42,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
return nil return nil
}) })
_ = context.CloseDatabase() context.CloseDatabase()
sort.Strings(repos) sort.Strings(repos)
@@ -70,13 +70,13 @@ func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len()) repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
i := 0 i := 0
_ = context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error { context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
repos[i] = repo repos[i] = repo
i++ i++
return nil return nil
}) })
_ = context.CloseDatabase() context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool { sort.Slice(repos, func(i, j int) bool {
return repos[i].Name < repos[j].Name return repos[i].Name < repos[j].Name
+2 -7
View File
@@ -61,11 +61,6 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
downloadUdebs = Yes downloadUdebs = Yes
} }
fmt.Printf("Download .udebs: %s\n", downloadUdebs) fmt.Printf("Download .udebs: %s\n", downloadUdebs)
downloadAppStream := No
if repo.DownloadAppStream {
downloadAppStream = Yes
}
fmt.Printf("Download AppStream: %s\n", downloadAppStream)
if repo.Filter != "" { if repo.Filter != "" {
fmt.Printf("Filter: %s\n", repo.Filter) fmt.Printf("Filter: %s\n", repo.Filter)
filterWithDeps := No filterWithDeps := No
@@ -91,7 +86,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
if repo.LastDownloadDate.IsZero() { if repo.LastDownloadDate.IsZero() {
fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n") fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
} else { } else {
_ = ListPackagesRefList(repo.RefList(), collectionFactory) ListPackagesRefList(repo.RefList(), collectionFactory)
} }
} }
@@ -124,7 +119,7 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
} }
list.PrepareIndex() list.PrepareIndex()
_ = list.ForEachIndexed(func(p *deb.Package) error { list.ForEachIndexed(func(p *deb.Package) error {
repo.Packages = append(repo.Packages, p.GetFullName()) repo.Packages = append(repo.Packages, p.GetFullName())
return nil return nil
}) })
+4 -15
View File
@@ -64,15 +64,6 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err) return fmt.Errorf("unable to update: %s", err)
} }
if repo.DownloadAppStream && !repo.IsFlat() {
context.Progress().Printf("Downloading AppStream metadata...\n")
err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
if repo.Filter != "" { if repo.Filter != "" {
context.Progress().Printf("Applying filter...\n") context.Progress().Printf("Applying filter...\n")
var filterQuery deb.PackageQuery var filterQuery deb.PackageQuery
@@ -96,11 +87,10 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
) )
skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool) skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
context.Progress().Printf("Building download queue...\n") context.Progress().Printf("Building download queue...\n")
queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly) collectionFactory.ChecksumCollection(nil), skipExistingPackages)
if err != nil { if err != nil {
return fmt.Errorf("unable to update: %s", err) return fmt.Errorf("unable to update: %s", err)
@@ -111,7 +101,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
err = context.ReOpenDatabase() err = context.ReOpenDatabase()
if err == nil { if err == nil {
repo.MarkAsIdle() repo.MarkAsIdle()
_ = collectionFactory.RemoteRepoCollection().Update(repo) collectionFactory.RemoteRepoCollection().Update(repo)
} }
}() }()
@@ -183,7 +173,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
file, e = os.CreateTemp("", task.File.Filename) file, e = os.CreateTemp("", task.File.Filename)
if e == nil { if e == nil {
task.TempDownPath = file.Name() task.TempDownPath = file.Name()
_ = file.Close() file.Close()
} }
} }
if e != nil { if e != nil {
@@ -271,7 +261,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n ")) return fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n "))
} }
_ = repo.FinalizeDownload(collectionFactory, context.Progress()) repo.FinalizeDownload(collectionFactory, context.Progress())
err = collectionFactory.RemoteRepoCollection().Update(repo) err = collectionFactory.RemoteRepoCollection().Update(repo)
if err != nil { if err != nil {
return fmt.Errorf("unable to update: %s", err) return fmt.Errorf("unable to update: %s", err)
@@ -302,7 +292,6 @@ Example:
cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata") cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures") cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror") cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)") cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)") cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error") cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
+1 -1
View File
@@ -40,7 +40,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
} }
format := context.Flags().Lookup("format").Value.String() format := context.Flags().Lookup("format").Value.String()
_ = PrintPackageList(result, format, "") PrintPackageList(result, format, "")
return err return err
} }
+3 -3
View File
@@ -84,8 +84,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
result := q.Query(collectionFactory.PackageCollection()) result := q.Query(collectionFactory.PackageCollection())
err = result.ForEach(func(p *deb.Package) error { err = result.ForEach(func(p *deb.Package) error {
_ = p.Stanza().WriteTo(w, p.IsSource, false, false) p.Stanza().WriteTo(w, p.IsSource, false, false)
_ = w.Flush() w.Flush()
fmt.Printf("\n") fmt.Printf("\n")
if withFiles { if withFiles {
@@ -109,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
if withReferences { if withReferences {
fmt.Printf("References to package:\n") fmt.Printf("References to package:\n")
_ = printReferencesTo(p, collectionFactory) printReferencesTo(p, collectionFactory)
fmt.Printf("\n") fmt.Printf("\n")
} }
+1 -33
View File
@@ -1,8 +1,6 @@
package cmd package cmd
import ( import (
"strings"
"github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/pgp"
"github.com/smira/commander" "github.com/smira/commander"
"github.com/smira/flag" "github.com/smira/flag"
@@ -14,20 +12,7 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
} }
signer := context.GetSigner() signer := context.GetSigner()
signer.SetKey(flags.Lookup("gpg-key").Value.String())
var gpgKeys []string
// CLI args have priority over config
cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
if len(cliKeys) > 0 {
gpgKeys = cliKeys
} else if len(context.Config().GpgKeys) > 0 {
gpgKeys = context.Config().GpgKeys
}
for _, gpgKey := range gpgKeys {
signer.SetKey(gpgKey)
}
signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String()) signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String()) signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
signer.SetBatch(flags.Lookup("batch").Value.Get().(bool)) signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -41,23 +26,6 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
} }
type gpgKeyFlag struct {
gpgKeys []string
}
func (k *gpgKeyFlag) Set(value string) error {
k.gpgKeys = append(k.gpgKeys, value)
return nil
}
func (k *gpgKeyFlag) Get() interface{} {
return k.gpgKeys
}
func (k *gpgKeyFlag) String() string {
return strings.Join(k.gpgKeys, ",")
}
func makeCmdPublish() *commander.Command { func makeCmdPublish() *commander.Command {
return &commander.Command{ return &commander.Command{
UsageLine: "publish", UsageLine: "publish",
+2 -2
View File
@@ -53,7 +53,7 @@ func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
return fmt.Errorf("unable to load list of repos: %s", err) return fmt.Errorf("unable to load list of repos: %s", err)
} }
_ = context.CloseDatabase() context.CloseDatabase()
sort.Strings(published) sort.Strings(published)
@@ -99,7 +99,7 @@ func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
return fmt.Errorf("unable to load list of repos: %s", err) return fmt.Errorf("unable to load list of repos: %s", err)
} }
_ = context.CloseDatabase() context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool { sort.Slice(repos, func(i, j int) bool {
return repos[i].GetPath() < repos[j].GetPath() return repos[i].GetPath() < repos[j].GetPath()
+1 -3
View File
@@ -34,7 +34,7 @@ Example:
} }
cmd.Flag.String("distribution", "", "distribution name to publish") cmd.Flag.String("distribution", "", "distribution name to publish")
cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)") cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -51,9 +51,7 @@ Example:
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)") cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
cmd.Flag.String("version", "", "version of the release")
return cmd return cmd
} }
+2 -12
View File
@@ -150,21 +150,13 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool) published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
} }
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("multi-dist") { if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
} }
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published) duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
if duplicate != nil { if duplicate != nil {
_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
} }
@@ -238,7 +230,7 @@ Example:
} }
cmd.Flag.String("distribution", "", "distribution name to publish") cmd.Flag.String("distribution", "", "distribution name to publish")
cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)") cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -255,8 +247,6 @@ Example:
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)") cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash") cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.String("version", "", "version of the release")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd return cmd
+1 -11
View File
@@ -99,14 +99,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool) published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
} }
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
if context.Flags().IsSet("multi-dist") { if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
} }
@@ -159,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh
`, `,
Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError), Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
} }
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -170,8 +162,6 @@ This command would switch published repository (with one component) named ppa/wh
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes") cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)") cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.String("version", "", "version of the release")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+1 -21
View File
@@ -60,26 +60,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool) published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
} }
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("origin") {
published.Origin = context.Flags().Lookup("origin").Value.String()
}
if context.Flags().IsSet("label") {
published.Label = context.Flags().Lookup("label").Value.String()
}
if context.Flags().IsSet("multi-dist") { if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool) published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
} }
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath()) err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
if err != nil { if err != nil {
return fmt.Errorf("unable to publish: %s", err) return fmt.Errorf("unable to publish: %s", err)
@@ -131,7 +115,7 @@ Example:
`, `,
Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError), Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
} }
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -141,12 +125,8 @@ Example:
cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes") cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes") cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch") cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component") cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions") cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
cmd.Flag.String("origin", "", "overwrite origin name to publish")
cmd.Flag.String("label", "", "overwrite label to publish")
cmd.Flag.String("version", "", "version of the release")
return cmd return cmd
} }
+4 -4
View File
@@ -32,7 +32,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
collectionFactory := context.NewCollectionFactory() collectionFactory := context.NewCollectionFactory()
repos := make([]string, collectionFactory.LocalRepoCollection().Len()) repos := make([]string, collectionFactory.LocalRepoCollection().Len())
i := 0 i := 0
_ = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
if raw { if raw {
repos[i] = repo.Name repos[i] = repo.Name
} else { } else {
@@ -47,7 +47,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
return nil return nil
}) })
_ = context.CloseDatabase() context.CloseDatabase()
sort.Strings(repos) sort.Strings(repos)
@@ -76,7 +76,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len()) repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
i := 0 i := 0
_ = context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error { context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo) e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
if e != nil { if e != nil {
return e return e
@@ -87,7 +87,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
return nil return nil
}) })
_ = context.CloseDatabase() context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool { sort.Slice(repos, func(i, j int) bool {
return repos[i].Name < repos[j].Name return repos[i].Name < repos[j].Name
+1 -1
View File
@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to remove: %s", err) return fmt.Errorf("unable to remove: %s", err)
} }
_ = toRemove.ForEach(func(p *deb.Package) error { toRemove.ForEach(func(p *deb.Package) error {
list.Remove(p) list.Remove(p)
context.Progress().ColoredPrintf("@r[-]@| %s removed", p) context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
return nil return nil
+1 -1
View File
@@ -52,7 +52,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool) withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
if withPackages { if withPackages {
_ = ListPackagesRefList(repo.RefList(), collectionFactory) ListPackagesRefList(repo.RefList(), collectionFactory)
} }
return err return err
+2 -2
View File
@@ -33,7 +33,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
collection := collectionFactory.SnapshotCollection() collection := collectionFactory.SnapshotCollection()
if raw { if raw {
_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error { collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
fmt.Printf("%s\n", snapshot.Name) fmt.Printf("%s\n", snapshot.Name)
return nil return nil
}) })
@@ -68,7 +68,7 @@ func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
jsonSnapshots := make([]*deb.Snapshot, collection.Len()) jsonSnapshots := make([]*deb.Snapshot, collection.Len())
i := 0 i := 0
_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error { collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
jsonSnapshots[i] = snapshot jsonSnapshots[i] = snapshot
i++ i++
return nil return nil
+2 -2
View File
@@ -116,7 +116,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
alreadySeen := map[string]bool{} alreadySeen := map[string]bool{}
_ = result.ForEachIndexed(func(pkg *deb.Package) error { result.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key] _, seen := alreadySeen[key]
@@ -132,7 +132,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
// If !allMatches, add only first matching name-arch package // If !allMatches, add only first matching name-arch package
if !seen || allMatches { if !seen || allMatches {
_ = packageList.Add(pkg) packageList.Add(pkg)
context.Progress().ColoredPrintf("@g[+]@| %s added", pkg) context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
} }
+1 -1
View File
@@ -123,7 +123,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
} }
format := context.Flags().Lookup("format").Value.String() format := context.Flags().Lookup("format").Value.String()
_ = PrintPackageList(result, format, "") PrintPackageList(result, format, "")
return err return err
} }
+2 -2
View File
@@ -79,7 +79,7 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool) withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
if withPackages { if withPackages {
_ = ListPackagesRefList(snapshot.RefList(), collectionFactory) ListPackagesRefList(snapshot.RefList(), collectionFactory)
} }
return err return err
@@ -139,7 +139,7 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
} }
list.PrepareIndex() list.PrepareIndex()
_ = list.ForEachIndexed(func(p *deb.Package) error { list.ForEachIndexed(func(p *deb.Package) error {
snapshot.Packages = append(snapshot.Packages, p.GetFullName()) snapshot.Packages = append(snapshot.Packages, p.GetFullName())
return nil return nil
}) })
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"os" "os"
"strings" "strings"
shellwords "github.com/mattn/go-shellwords" "github.com/mattn/go-shellwords"
"github.com/smira/commander" "github.com/smira/commander"
) )
@@ -31,7 +31,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = file.Close() }() defer file.Close()
scanner := bufio.NewScanner(file) scanner := bufio.NewScanner(file)
-3
View File
@@ -185,7 +185,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
$keyring \ $keyring \
"-with-sources=[download source packages in addition to binary packages]:$bool" \ "-with-sources=[download source packages in addition to binary packages]:$bool" \
"-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \ "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
"-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
"(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components" "(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
;; ;;
list) list)
@@ -212,7 +211,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
$keyring \ $keyring \
"-max-tries=[max download tries till process fails with download error]:number: " \ "-max-tries=[max download tries till process fails with download error]:number: " \
"-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \ "-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
"-latest=[download only latest version of each package (per architecture)]:$bool" \
"(-)2:mirror name:$mirrors" "(-)2:mirror name:$mirrors"
;; ;;
rename) rename)
@@ -225,7 +223,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
"-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \ "-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
"-with-sources=[download source packages in addition to binary packages]:$bool" \ "-with-sources=[download source packages in addition to binary packages]:$bool" \
"-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \ "-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
"-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
"(-)2:mirror name:$mirrors" "(-)2:mirror name:$mirrors"
;; ;;
search) search)
+12 -48
View File
@@ -22,36 +22,34 @@
__aptly_mirror_list() __aptly_mirror_list()
{ {
aptly ${aptly_global_opts[@]} mirror list -raw aptly mirror list -raw
} }
__aptly_repo_list() __aptly_repo_list()
{ {
aptly ${aptly_global_opts[@]} repo list -raw aptly repo list -raw
} }
__aptly_snapshot_list() __aptly_snapshot_list()
{ {
aptly ${aptly_global_opts[@]} snapshot list -raw aptly snapshot list -raw
} }
__aptly_published_distributions() __aptly_published_distributions()
{ {
aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 2 | sort | uniq aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
} }
__aptly_published_prefixes() __aptly_published_prefixes()
{ {
aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 1 | sort | uniq aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
} }
__aptly_prefixes_for_distribution() __aptly_prefixes_for_distribution()
{ {
aptly ${aptly_global_opts[@]} publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
} }
_aptly() _aptly()
{ {
cur="${COMP_WORDS[COMP_CWORD]}" cur="${COMP_WORDS[COMP_CWORD]}"
@@ -59,12 +57,7 @@ _aptly()
prevprev="${COMP_WORDS[COMP_CWORD-2]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}"
commands="api config db graph mirror package publish repo serve snapshot task version" commands="api config db graph mirror package publish repo serve snapshot task version"
options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
options="-architectures -config -db-open-attempts -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider"
options_without_arg="-dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve"
options_with_arg="-architectures -db-open-attempts -gpg-provider"
options_with_path_arg="-config"
db_subcommands="cleanup recover" db_subcommands="cleanup recover"
mirror_subcommands="create drop edit show list rename search update" mirror_subcommands="create drop edit show list rename search update"
publish_subcommands="drop list repo snapshot switch update source" publish_subcommands="drop list repo snapshot switch update source"
@@ -76,41 +69,12 @@ _aptly()
config_subcommands="show" config_subcommands="show"
api_subcommands="serve" api_subcommands="serve"
local cmd subcmd numargs numoptions i aptly_global_opts local cmd subcmd numargs numoptions i
numargs=0 numargs=0
numoptions=0 numoptions=0
for opt in "${options_with_path_arg[@]}"; do
[[ "$prev" == "$opt" ]] || continue
compopt -o filenames 2>/dev/null
_filedir
return 0
done
for (( i=1; i < $COMP_CWORD; i++ )); do for (( i=1; i < $COMP_CWORD; i++ )); do
word=${COMP_WORDS[i]}
if [[ "$word" == -*=* ]]; then
for o in "${options[@]}"; do
[[ ${word%%=*} == "$o" ]] && aptly_global_opts+=("$word")
done
else
for o in "${options_with_arg[@]}" ""${options_with_path_arg[@]}"" ; do
if [[ "$word" == "$o" ]]; then
if (( i + 1 < COMP_CWORD )); then
aptly_global_opts+=("$word" "${COMP_WORDS[i+1]}")
else
aptly_global_opts+=("$word")
fi
(( i++ ))
continue 2
fi
done
fi
for o in ${options_without_arg[@]}; do
[[ "$word" == "$o" ]] && aptly_global_opts+=("$word")
done
if [[ -n "$cmd" ]]; then if [[ -n "$cmd" ]]; then
if [[ ! -n "$subcmd" ]]; then if [[ ! -n "$subcmd" ]]; then
subcmd=${COMP_WORDS[i]} subcmd=${COMP_WORDS[i]}
@@ -203,7 +167,7 @@ _aptly()
"create") "create")
if [[ $numargs -eq 0 ]]; then if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur})) COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
return 0 return 0
fi fi
fi fi
@@ -211,7 +175,7 @@ _aptly()
"edit") "edit")
if [[ $numargs -eq 0 ]]; then if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur})) COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
else else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi fi
@@ -263,7 +227,7 @@ _aptly()
"update") "update")
if [[ $numargs -eq 0 ]]; then if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur})) COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
else else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur})) COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi fi
@@ -375,7 +339,7 @@ _aptly()
if [[ "$cur" == -* ]]; then if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur})) COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
else else
compopt -o filenames 2>/dev/null comptopt -o filenames 2>/dev/null
COMPREPLY=($(compgen -f -- ${cur})) COMPREPLY=($(compgen -f -- ${cur}))
return 0 return 0
fi fi
+2 -3
View File
@@ -6,7 +6,6 @@ import (
"strings" "strings"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/cheggaaa/pb" "github.com/cheggaaa/pb"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"github.com/wsxiaoys/terminal/color" "github.com/wsxiaoys/terminal/color"
@@ -79,7 +78,7 @@ func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
if p.bar != nil { if p.bar != nil {
panic("bar already initialized") panic("bar already initialized")
} }
if utils.RunningOnTerminal() { if RunningOnTerminal() {
p.bar = pb.New(0) p.bar = pb.New(0)
p.bar.Total = count p.bar.Total = count
p.bar.NotPrint = true p.bar.NotPrint = true
@@ -142,7 +141,7 @@ func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
// ColoredPrintf does printf in colored way + newline // ColoredPrintf does printf in colored way + newline
func (p *Progress) ColoredPrintf(msg string, a ...interface{}) { func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
if utils.RunningOnTerminal() { if RunningOnTerminal() {
p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"} p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
} else { } else {
// stip color marks // stip color marks
+1 -1
View File
@@ -11,7 +11,7 @@ func Test(t *testing.T) {
TestingT(t) TestingT(t)
} }
type ProgressSuite struct{} type ProgressSuite struct {}
var _ = Suite(&ProgressSuite{}) var _ = Suite(&ProgressSuite{})
+12
View File
@@ -0,0 +1,12 @@
package console
import (
"syscall"
"golang.org/x/term"
)
// RunningOnTerminal checks whether stdout is terminal
func RunningOnTerminal() bool {
return term.IsTerminal(syscall.Stdout)
}
+13 -22
View File
@@ -100,7 +100,6 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"} configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
for _, configLocation := range configLocations { for _, configLocation := range configLocations {
// FIXME: check if exists, check if readable
err = utils.LoadConfig(configLocation, &utils.Config) err = utils.LoadConfig(configLocation, &utils.Config)
if os.IsPermission(err) || os.IsNotExist(err) { if os.IsPermission(err) || os.IsNotExist(err) {
continue continue
@@ -116,7 +115,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation) fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf) utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
err = utils.LoadConfig(homeLocation, &utils.Config) err = utils.LoadConfig(homeLocation, &utils.Config)
if err != nil { if err != nil {
Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err)) Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -124,14 +123,6 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
} }
} }
if utils.Config.LogFormat == "json" {
context.StructuredLogging(true)
utils.SetupJSONLogger(utils.Config.LogLevel, os.Stdout)
} else {
context.StructuredLogging(false)
utils.SetupDefaultLogger(utils.Config.LogLevel)
}
context.configLoaded = true context.configLoaded = true
} }
@@ -242,7 +233,7 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo
// If flag is defined prefer it to global setting // If flag is defined prefer it to global setting
maxTries = maxTriesFlag.Value.Get().(int) maxTries = maxTriesFlag.Value.Get().(int)
} }
var downloader = context.config().Downloader var downloader string = context.config().Downloader
downloaderFlag := context.flags.Lookup("downloader") downloaderFlag := context.flags.Lookup("downloader")
if downloaderFlag != nil { if downloaderFlag != nil {
downloader = downloaderFlag.Value.String() downloader = downloaderFlag.Value.String()
@@ -304,8 +295,8 @@ func (context *AptlyContext) _database() (database.Storage, error) {
switch context.config().DatabaseBackend.Type { switch context.config().DatabaseBackend.Type {
case "leveldb": case "leveldb":
dbPath := filepath.Join(context.config().GetRootDir(), "db") dbPath := filepath.Join(context.config().GetRootDir(), "db")
if len(context.config().DatabaseBackend.DBPath) != 0 { if len(context.config().DatabaseBackend.DbPath) != 0 {
dbPath = context.config().DatabaseBackend.DBPath dbPath = context.config().DatabaseBackend.DbPath
} }
context.database, err = goleveldb.NewDB(dbPath) context.database, err = goleveldb.NewDB(dbPath)
case "etcd": case "etcd":
@@ -453,7 +444,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
} else if strings.HasPrefix(name, "azure:") { } else if strings.HasPrefix(name, "azure:") {
params, ok := context.config().AzurePublishRoots[name[6:]] params, ok := context.config().AzurePublishRoots[name[6:]]
if !ok { if !ok {
Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:])) Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:]))
} }
var err error var err error
@@ -598,17 +589,17 @@ func (context *AptlyContext) Shutdown() {
if aptly.EnableDebug { if aptly.EnableDebug {
if context.fileMemProfile != nil { if context.fileMemProfile != nil {
_ = pprof.WriteHeapProfile(context.fileMemProfile) pprof.WriteHeapProfile(context.fileMemProfile)
_ = context.fileMemProfile.Close() context.fileMemProfile.Close()
context.fileMemProfile = nil context.fileMemProfile = nil
} }
if context.fileCPUProfile != nil { if context.fileCPUProfile != nil {
pprof.StopCPUProfile() pprof.StopCPUProfile()
_ = context.fileCPUProfile.Close() context.fileCPUProfile.Close()
context.fileCPUProfile = nil context.fileCPUProfile = nil
} }
if context.fileMemProfile != nil { if context.fileMemProfile != nil {
_ = context.fileMemProfile.Close() context.fileMemProfile.Close()
context.fileMemProfile = nil context.fileMemProfile = nil
} }
} }
@@ -616,7 +607,7 @@ func (context *AptlyContext) Shutdown() {
context.taskList.Stop() context.taskList.Stop()
} }
if context.database != nil { if context.database != nil {
_ = context.database.Close() context.database.Close()
context.database = nil context.database = nil
} }
if context.downloader != nil { if context.downloader != nil {
@@ -661,7 +652,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
_ = pprof.StartCPUProfile(context.fileCPUProfile) pprof.StartCPUProfile(context.fileCPUProfile)
} }
memprofile := flags.Lookup("memprofile").Value.String() memprofile := flags.Lookup("memprofile").Value.String()
@@ -681,7 +672,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
return nil, err return nil, err
} }
_, _ = context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n") context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
go func() { go func() {
var stats runtime.MemStats var stats runtime.MemStats
@@ -691,7 +682,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
for { for {
runtime.ReadMemStats(&stats) runtime.ReadMemStats(&stats)
if context.fileMemStats != nil { if context.fileMemStats != nil {
_, _ = context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n", context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased)) (time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
time.Sleep(interval) time.Sleep(interval)
} else { } else {
+11 -11
View File
@@ -14,7 +14,8 @@ func Test(t *testing.T) {
} }
type EtcDDBSuite struct { type EtcDDBSuite struct {
db database.Storage url string
db database.Storage
} }
var _ = Suite(&EtcDDBSuite{}) var _ = Suite(&EtcDDBSuite{})
@@ -66,17 +67,17 @@ func (s *EtcDDBSuite) TestDelete(c *C) {
func (s *EtcDDBSuite) TestByPrefix(c *C) { func (s *EtcDDBSuite) TestByPrefix(c *C) {
//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{}) //c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03}) s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02}) s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04}) s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05}) s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
@@ -108,7 +109,7 @@ func (s *EtcDDBSuite) TestHasPrefix(c *C) {
//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false) //c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false) //c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true) c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true) c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -123,17 +124,15 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
value2 = []byte("value2") value2 = []byte("value2")
) )
transaction, err := s.db.OpenTransaction() transaction, err := s.db.OpenTransaction()
c.Assert(err, IsNil)
err = s.db.Put(key, value) err = s.db.Put(key, value)
c.Assert(err, IsNil) c.Assert(err, IsNil)
c.Assert(err, IsNil) c.Assert(err, IsNil)
_ = transaction.Put(key2, value2) transaction.Put(key2, value2)
v, err := s.db.Get(key) v, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Check(v, DeepEquals, value) c.Check(v, DeepEquals, value)
err = transaction.Delete(key) err = transaction.Delete(key)
c.Assert(err, IsNil) c.Assert(err, IsNil)
_, err = transaction.Get(key2) _, err = transaction.Get(key2)
@@ -156,3 +155,4 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
_, err = transaction.Get(key) _, err = transaction.Get(key)
c.Assert(err, NotNil) c.Assert(err, NotNil)
} }
+1 -1
View File
@@ -145,7 +145,7 @@ func (s *EtcDStorage) Close() error {
return err return err
} }
// Open returns the database // Reopen tries to open (re-open) the database
func (s *EtcDStorage) Open() error { func (s *EtcDStorage) Open() error {
if s.db != nil { if s.db != nil {
return nil return nil
+2 -1
View File
@@ -67,7 +67,8 @@ func (t *transaction) Commit() (err error) {
// Discard is safe to call after Commit(), it would be no-op // Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() { func (t *transaction) Discard() {
t.ops = []clientv3.Op{} t.ops = []clientv3.Op{}
_ = t.tmpdb.Drop() t.tmpdb.Drop()
return
} }
// transaction should implement database.Transaction // transaction should implement database.Transaction
+2 -2
View File
@@ -51,8 +51,8 @@ func RecoverDB(path string) error {
return err return err
} }
_ = db.Close() db.Close()
_ = stor.Close() stor.Close()
return nil return nil
} }
+11 -11
View File
@@ -119,17 +119,17 @@ func (s *LevelDBSuite) TestDelete(c *C) {
func (s *LevelDBSuite) TestByPrefix(c *C) { func (s *LevelDBSuite) TestByPrefix(c *C) {
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03}) s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02}) s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04}) s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05}) s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}}) c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}) c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
@@ -161,7 +161,7 @@ func (s *LevelDBSuite) TestHasPrefix(c *C) {
c.Check(s.db.HasPrefix([]byte(nil)), Equals, false) c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false) c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true) c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true) c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -180,8 +180,8 @@ func (s *LevelDBSuite) TestBatch(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
batch := s.db.CreateBatch() batch := s.db.CreateBatch()
_ = batch.Put(key2, value2) batch.Put(key2, value2)
_ = batch.Delete(key) batch.Delete(key)
v, err := s.db.Get(key) v, err := s.db.Get(key)
c.Check(err, IsNil) c.Check(err, IsNil)
@@ -202,9 +202,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
} }
func (s *LevelDBSuite) TestCompactDB(c *C) { func (s *LevelDBSuite) TestCompactDB(c *C) {
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01}) s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03}) s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02}) s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.CompactDB(), IsNil) c.Check(s.db.CompactDB(), IsNil)
} }
+8 -8
View File
@@ -60,14 +60,14 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = input.Close() }() defer input.Close()
isClearSigned, err := verifier.IsClearSigned(input) isClearSigned, err := verifier.IsClearSigned(input)
if err != nil { if err != nil {
return err return err
} }
_, _ = input.Seek(0, 0) input.Seek(0, 0)
if !isClearSigned && !acceptUnsigned { if !isClearSigned && !acceptUnsigned {
return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled") return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
@@ -79,7 +79,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil { if err != nil {
return err return err
} }
_, _ = input.Seek(0, 0) input.Seek(0, 0)
c.SignatureKeys = keyInfo.GoodKeys c.SignatureKeys = keyInfo.GoodKeys
} }
@@ -91,7 +91,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = text.Close() }() defer text.Close()
} else { } else {
text = input text = input
} }
@@ -307,7 +307,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil { if err != nil {
failedFiles = append(failedFiles, path) failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err) reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup() changes.Cleanup()
continue continue
} }
@@ -315,7 +315,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil { if err != nil {
failedFiles = append(failedFiles, path) failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err) reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup() changes.Cleanup()
continue continue
} }
@@ -334,7 +334,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil { if err != nil {
failedFiles = append(failedFiles, path) failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err) reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup() changes.Cleanup()
continue continue
} }
@@ -354,7 +354,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
failedFiles = append(failedFiles, path) failedFiles = append(failedFiles, path)
reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s", reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
changes.ChangesName, changes.SignatureKeys, err) changes.ChangesName, changes.SignatureKeys, err)
_ = changes.Cleanup() changes.Cleanup()
continue continue
} }
} }
+3 -3
View File
@@ -51,7 +51,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
func (s *ChangesSuite) TearDownTest(c *C) { func (s *ChangesSuite) TearDownTest(c *C) {
s.progress.Shutdown() s.progress.Shutdown()
_ = s.db.Close() s.db.Close()
} }
func (s *ChangesSuite) TestParseAndVerify(c *C) { func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -108,13 +108,13 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
for _, path := range origFailedFiles { for _, path := range origFailedFiles {
filename := filepath.Join(s.Dir, filepath.Base(path)) filename := filepath.Join(s.Dir, filepath.Base(path))
_ = utils.CopyFile(path, filename) utils.CopyFile(path, filename)
expectedFailedFiles = append(expectedFailedFiles, filename) expectedFailedFiles = append(expectedFailedFiles, filename)
} }
for _, path := range origProcessedFiles { for _, path := range origProcessedFiles {
filename := filepath.Join(s.Dir, filepath.Base(path)) filename := filepath.Join(s.Dir, filepath.Base(path))
_ = utils.CopyFile(path, filename) utils.CopyFile(path, filename)
expectedProcessedFiles = append(expectedProcessedFiles, filename) expectedProcessedFiles = append(expectedProcessedFiles, filename)
} }
+1 -1
View File
@@ -28,7 +28,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
} }
func (s *ChecksumCollectionSuite) TearDownTest(c *C) { func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *ChecksumCollectionSuite) TestFlow(c *C) { func (s *ChecksumCollectionSuite) TestFlow(c *C) {
+10 -10
View File
@@ -17,7 +17,7 @@ import (
"github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/pgp"
"github.com/kjk/lzma" "github.com/kjk/lzma"
"github.com/klauspost/compress/zstd" "github.com/klauspost/compress/zstd"
xz "github.com/smira/go-xz" "github.com/smira/go-xz"
) )
// Source kinds // Source kinds
@@ -35,7 +35,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() { _ = file.Close() }() defer file.Close()
library := ar.NewReader(file) library := ar.NewReader(file)
for { for {
@@ -66,14 +66,14 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile) return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
} }
defer func() { _ = ungzip.Close() }() defer ungzip.Close()
tarInput = ungzip tarInput = ungzip
case "control.tar.xz": case "control.tar.xz":
unxz, err := xz.NewReader(bufReader) unxz, err := xz.NewReader(bufReader)
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile) return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
} }
defer func() { _ = unxz.Close() }() defer unxz.Close()
tarInput = unxz tarInput = unxz
case "control.tar.zst": case "control.tar.zst":
unzstd, err := zstd.NewReader(bufReader) unzstd, err := zstd.NewReader(bufReader)
@@ -116,10 +116,10 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() { _ = file.Close() }() defer file.Close()
isClearSigned, err := verifier.IsClearSigned(file) isClearSigned, err := verifier.IsClearSigned(file)
_, _ = file.Seek(0, 0) file.Seek(0, 0)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -132,7 +132,7 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() { _ = text.Close() }() defer text.Close()
} else { } else {
text = file text = file
} }
@@ -181,7 +181,7 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile) return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
} }
defer func() { _ = ungzip.Close() }() defer ungzip.Close()
tarInput = ungzip tarInput = ungzip
} }
case "data.tar.bz2": case "data.tar.bz2":
@@ -191,11 +191,11 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
if err != nil { if err != nil {
return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile) return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
} }
defer func() { _ = unxz.Close() }() defer unxz.Close()
tarInput = unxz tarInput = unxz
case "data.tar.lzma": case "data.tar.lzma":
unlzma := lzma.NewReader(bufReader) unlzma := lzma.NewReader(bufReader)
defer func() { _ = unlzma.Close() }() defer unlzma.Close()
tarInput = unlzma tarInput = unlzma
case "data.tar.zst": case "data.tar.zst":
unzstd, err := zstd.NewReader(bufReader) unzstd, err := zstd.NewReader(bufReader)
-45
View File
@@ -1,45 +0,0 @@
package deb
import (
"errors"
"fmt"
"github.com/aptly-dev/aptly/database"
)
// FindDanglingReferences finds references that exist in the given PackageRefList, but not in the given PackageCollection.
// It returns all such references, so they can be removed from the database.
func FindDanglingReferences(reflist *PackageRefList, packages *PackageCollection) (dangling *PackageRefList, err error) {
dangling = &PackageRefList{}
err = reflist.ForEach(func(key []byte) error {
ok, err := isDangling(packages, key)
if err != nil {
return err
}
if ok {
dangling.Refs = append(dangling.Refs, key)
}
return nil
})
if err != nil {
return nil, err
}
return dangling, nil
}
func isDangling(packages *PackageCollection, key []byte) (bool, error) {
_, err := packages.ByKey(key)
if errors.Is(err, database.ErrNotFound) {
return true, nil
}
if err != nil {
return false, fmt.Errorf("get reference %q: %w", key, err)
}
return false, nil
}
-46
View File
@@ -1,46 +0,0 @@
package deb_test
import (
"bytes"
"testing"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/deb"
)
func TestFindDanglingReferences(t *testing.T) {
reflist := deb.NewPackageRefList()
reflist.Refs = [][]byte{[]byte("P existing 1.2.3"), []byte("P dangling 1.2.3")}
db, _ := goleveldb.NewOpenDB(t.TempDir())
packages := deb.NewPackageCollection(db)
if err := packages.Update(&deb.Package{Name: "existing", Version: "1.2.3"}); err != nil {
t.Fatal(err)
}
dangling, err := deb.FindDanglingReferences(reflist, packages)
if err != nil {
t.Fatal(err)
}
exp := &deb.PackageRefList{
Refs: [][]byte{[]byte("P dangling 1.2.3")},
}
compareRefs(t, exp, dangling)
}
func compareRefs(t *testing.T, exp, got *deb.PackageRefList) {
t.Helper()
if len(exp.Refs) != len(got.Refs) {
t.Fatalf("refs length mismatch: exp %d, got %d", len(exp.Refs), len(got.Refs))
}
for i := range exp.Refs {
if !bytes.Equal(exp.Refs[i], got.Refs[i]) {
t.Fatalf("refs do not match: exp %q, got %q", exp.Refs[i], got.Refs[i])
}
}
}
-3
View File
@@ -26,11 +26,8 @@ var (
"Version", "Version",
"Codename", "Codename",
"Date", "Date",
"Valid-Until",
"NotAutomatic", "NotAutomatic",
"ButAutomaticUpgrades", "ButAutomaticUpgrades",
"Acquire-By-Hash",
"Signed-By",
"Architectures", "Architectures",
"Architecture", "Architecture",
"Components", "Components",
+1 -1
View File
@@ -163,7 +163,7 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
func (s *ControlFileSuite) TestLongFields(c *C) { func (s *ControlFileSuite) TestLongFields(c *C) {
f, err := os.Open("long.stanza") f, err := os.Open("long.stanza")
c.Assert(err, IsNil) c.Assert(err, IsNil)
defer func() { _ = f.Close() }() defer f.Close()
r := NewControlFileReader(f, false, false) r := NewControlFileReader(f, false, false)
stanza, e := r.ReadStanza() stanza, e := r.ReadStanza()
+11 -11
View File
@@ -12,15 +12,15 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
var err error var err error
graph := gographviz.NewEscape() graph := gographviz.NewEscape()
_ = graph.SetDir(true) graph.SetDir(true)
_ = graph.SetName("aptly") graph.SetName("aptly")
var labelStart string var labelStart string
var labelEnd string var labelEnd string
switch layout { switch layout {
case "vertical": case "vertical":
_ = graph.AddAttr("aptly", "rankdir", "LR") graph.AddAttr("aptly", "rankdir", "LR")
labelStart = "" labelStart = ""
labelEnd = "" labelEnd = ""
case "horizontal": case "horizontal":
@@ -38,7 +38,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return e return e
} }
_ = graph.AddNode("aptly", repo.UUID, map[string]string{ graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord", "shape": "Mrecord",
"style": "filled", "style": "filled",
"fillcolor": "darkgoldenrod1", "fillcolor": "darkgoldenrod1",
@@ -60,7 +60,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return e return e
} }
_ = graph.AddNode("aptly", repo.UUID, map[string]string{ graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord", "shape": "Mrecord",
"style": "filled", "style": "filled",
"fillcolor": "mediumseagreen", "fillcolor": "mediumseagreen",
@@ -75,7 +75,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return nil, err return nil, err
} }
_ = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error { collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
existingNodes[snapshot.UUID] = true existingNodes[snapshot.UUID] = true
return nil return nil
}) })
@@ -91,7 +91,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
description = "Snapshot from repo" description = "Snapshot from repo"
} }
_ = graph.AddNode("aptly", snapshot.UUID, map[string]string{ graph.AddNode("aptly", snapshot.UUID, map[string]string{
"shape": "Mrecord", "shape": "Mrecord",
"style": "filled", "style": "filled",
"fillcolor": "cadetblue1", "fillcolor": "cadetblue1",
@@ -103,7 +103,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
for _, uuid := range snapshot.SourceIDs { for _, uuid := range snapshot.SourceIDs {
_, exists := existingNodes[uuid] _, exists := existingNodes[uuid]
if exists { if exists {
_ = graph.AddEdge(uuid, snapshot.UUID, true, nil) graph.AddEdge(uuid, snapshot.UUID, true, nil)
} }
} }
} }
@@ -114,8 +114,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return nil, err return nil, err
} }
_ = collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error { collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
_ = graph.AddNode("aptly", repo.UUID, map[string]string{ graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord", "shape": "Mrecord",
"style": "filled", "style": "filled",
"fillcolor": "darkolivegreen1", "fillcolor": "darkolivegreen1",
@@ -127,7 +127,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
for _, uuid := range repo.Sources { for _, uuid := range repo.Sources {
_, exists := existingNodes[uuid] _, exists := existingNodes[uuid]
if exists { if exists {
_ = graph.AddEdge(uuid, repo.UUID, true, nil) graph.AddEdge(uuid, repo.UUID, true, nil)
} }
} }
+7 -7
View File
@@ -59,24 +59,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
if file.discardable { if file.discardable {
return nil return nil
} }
_, _ = file.BufWriter() file.BufWriter()
} }
err := file.w.Flush() err := file.w.Flush()
if err != nil { if err != nil {
_ = file.tempFile.Close() file.tempFile.Close()
return fmt.Errorf("unable to write to index file: %s", err) return fmt.Errorf("unable to write to index file: %s", err)
} }
if file.compressable { if file.compressable {
err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2) err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
if err != nil { if err != nil {
_ = file.tempFile.Close() file.tempFile.Close()
return fmt.Errorf("unable to compress index file: %s", err) return fmt.Errorf("unable to compress index file: %s", err)
} }
} }
_ = file.tempFile.Close() file.tempFile.Close()
exts := []string{""} exts := []string{""}
cksumExts := exts cksumExts := exts
@@ -220,11 +220,11 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
// If we managed to resolve the link target: delete it. This is the // If we managed to resolve the link target: delete it. This is the
// oldest physical index file we no longer need. Once we drop our // oldest physical index file we no longer need. Once we drop our
// old symlink we'll essentially forget about it existing at all. // old symlink we'll essentially forget about it existing at all.
_ = file.parent.publishedStorage.Remove(linkTarget) file.parent.publishedStorage.Remove(linkTarget)
} }
_ = file.parent.publishedStorage.Remove(oldIndexPath) file.parent.publishedStorage.Remove(oldIndexPath)
} }
_ = file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath) file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
} }
// create symlink // create symlink
+2 -36
View File
@@ -172,39 +172,6 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
return err return err
} }
// FilterLatest creates a copy of the package list containing only the
// latest version for each package name/architecture pair.
func (l *PackageList) FilterLatest() (*PackageList, error) {
if l == nil {
return nil, fmt.Errorf("package list is nil")
}
filtered := make(map[string]*Package, l.Len())
err := l.ForEach(func(p *Package) error {
key := p.Architecture + "|" + p.Name
if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 {
filtered[key] = p
}
return nil
})
if err != nil {
return nil, err
}
result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
for _, pkg := range filtered {
if err = result.Add(pkg); err != nil {
return nil, err
}
}
return result, nil
}
// ForEachIndexed calls handler for each package in list in indexed order // ForEachIndexed calls handler for each package in list in indexed order
func (l *PackageList) ForEachIndexed(handler func(*Package) error) error { func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
if !l.indexed { if !l.indexed {
@@ -471,7 +438,7 @@ func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0) result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
for _, pkg := range l.packages { for _, pkg := range l.packages {
if q.Matches(pkg) { if q.Matches(pkg) {
_ = result.Add(pkg) result.Add(pkg)
} }
} }
@@ -489,7 +456,7 @@ func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageLi
pkg := l.packages["P"+arch+" "+name+" "+version] pkg := l.packages["P"+arch+" "+name+" "+version]
if pkg != nil { if pkg != nil {
_ = result.Add(pkg) result.Add(pkg)
} }
return return
@@ -631,7 +598,6 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) {
// //
// when follow-all-variants is enabled, we need to try to expand anyway, // when follow-all-variants is enabled, we need to try to expand anyway,
// as even if dependency is satisfied now, there might be other ways to satisfy dependency // as even if dependency is satisfied now, there might be other ways to satisfy dependency
// FIXME: do not search twice
if result.Search(dep, false, true) != nil { if result.Search(dep, false, true) != nil {
if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil {
options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true))
+10 -59
View File
@@ -96,7 +96,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}}, {Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
} }
for _, p := range s.packages { for _, p := range s.packages {
_ = s.il.Add(p) s.il.Add(p)
} }
s.il.PrepareIndex() s.il.PrepareIndex()
@@ -110,7 +110,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}}, {Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
} }
for _, p := range s.packages2 { for _, p := range s.packages2 {
_ = s.il2.Add(p) s.il2.Add(p)
} }
s.il2.PrepareIndex() s.il2.PrepareIndex()
@@ -202,8 +202,8 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
} }
func (s *PackageListSuite) TestForeach(c *C) { func (s *PackageListSuite) TestForeach(c *C) {
_ = s.list.Add(s.p1) s.list.Add(s.p1)
_ = s.list.Add(s.p3) s.list.Add(s.p3)
Len := 0 Len := 0
err := s.list.ForEach(func(*Package) error { err := s.list.ForEach(func(*Package) error {
@@ -232,21 +232,21 @@ func (s *PackageListSuite) TestIndex(c *C) {
} }
func (s *PackageListSuite) TestAppend(c *C) { func (s *PackageListSuite) TestAppend(c *C) {
_ = s.list.Add(s.p1) s.list.Add(s.p1)
_ = s.list.Add(s.p3) s.list.Add(s.p3)
err := s.list.Append(s.il) err := s.list.Append(s.il)
c.Check(err, IsNil) c.Check(err, IsNil)
c.Check(s.list.Len(), Equals, 16) c.Check(s.list.Len(), Equals, 16)
list := NewPackageList() list := NewPackageList()
_ = list.Add(s.p4) list.Add(s.p4)
err = s.list.Append(list) err = s.list.Append(list)
c.Check(err, ErrorMatches, "package already exists and is different: .*") c.Check(err, ErrorMatches, "package already exists and is different: .*")
s.list.PrepareIndex() s.list.PrepareIndex()
c.Check(func() { _ = s.list.Append(s.il) }, Panics, "Append not supported when indexed") c.Check(func() { s.list.Append(s.il) }, Panics, "Append not supported when indexed")
} }
func (s *PackageListSuite) TestSearch(c *C) { func (s *PackageListSuite) TestSearch(c *C) {
@@ -312,7 +312,7 @@ func (s *PackageListSuite) TestSearch(c *C) {
func (s *PackageListSuite) TestFilter(c *C) { func (s *PackageListSuite) TestFilter(c *C) {
c.Check(func() { c.Check(func() {
_, _ = s.list.Filter(FilterOptions{ s.list.Filter(FilterOptions{
Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}}, Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}},
}) })
}, Panics, "list not indexed, can't filter") }, Panics, "list not indexed, can't filter")
@@ -479,7 +479,7 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}}) {Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
for _, p := range s.sourcePackages { for _, p := range s.sourcePackages {
_ = s.il.Add(p) s.il.Add(p)
} }
missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil) missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
@@ -503,52 +503,3 @@ func (s *PackageListSuite) TestArchitectures(c *C) {
sort.Strings(archs) sort.Strings(archs)
c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"}) c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
} }
func (s *PackageListSuite) TestFilterLatest(c *C) {
list := NewPackageList()
older := packageStanza.Copy()
older["Version"] = "1.0"
olderPkg := NewPackageFromControlFile(older)
_ = list.Add(olderPkg)
newer := packageStanza.Copy()
newer["Version"] = "2.0"
newerPkg := NewPackageFromControlFile(newer)
_ = list.Add(newerPkg)
shared := packageStanza.Copy()
shared["Architecture"] = ArchitectureAll
shared["Version"] = "3.0"
shared["Package"] = "shared"
sharedPkg := NewPackageFromControlFile(shared)
_ = list.Add(sharedPkg)
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.Len(), Equals, 2)
c.Check(filtered.Has(newerPkg), Equals, true)
c.Check(filtered.Has(sharedPkg), Equals, true)
}
func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
list := NewPackageListWithDuplicates(true, 2)
_ = list.Add(NewPackageFromControlFile(packageStanza.Copy()))
another := packageStanza.Copy()
another["Version"] = "7.41-1"
_ = list.Add(NewPackageFromControlFile(another))
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.duplicatesAllowed, Equals, true)
}
func (s *PackageListSuite) TestFilterLatestNil(c *C) {
var list *PackageList
filtered, err := list.FilterLatest()
c.Assert(err, ErrorMatches, "package list is nil")
c.Assert(filtered, IsNil)
}
+6 -6
View File
@@ -69,7 +69,7 @@ func (repo *LocalRepo) Encode() []byte {
var buf bytes.Buffer var buf bytes.Buffer
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
_ = encoder.Encode(repo) encoder.Encode(repo)
return buf.Bytes() return buf.Bytes()
} }
@@ -116,7 +116,7 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
return result return result
} }
_ = collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error { collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
r := &LocalRepo{} r := &LocalRepo{}
if err := r.Decode(blob); err != nil { if err := r.Decode(blob); err != nil {
log.Printf("Error decoding local repo: %s\n", err) log.Printf("Error decoding local repo: %s\n", err)
@@ -159,9 +159,9 @@ func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
// Update stores updated information about repo in DB // Update stores updated information about repo in DB
func (collection *LocalRepoCollection) Update(repo *LocalRepo) error { func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
batch := collection.db.CreateBatch() batch := collection.db.CreateBatch()
_ = batch.Put(repo.Key(), repo.Encode()) batch.Put(repo.Key(), repo.Encode())
if repo.packageRefs != nil { if repo.packageRefs != nil {
_ = batch.Put(repo.RefKey(), repo.packageRefs.Encode()) batch.Put(repo.RefKey(), repo.packageRefs.Encode())
} }
return batch.Write() return batch.Write()
} }
@@ -247,7 +247,7 @@ func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
delete(collection.cache, repo.UUID) delete(collection.cache, repo.UUID)
batch := collection.db.CreateBatch() batch := collection.db.CreateBatch()
_ = batch.Delete(repo.Key()) batch.Delete(repo.Key())
_ = batch.Delete(repo.RefKey()) batch.Delete(repo.RefKey())
return batch.Write() return batch.Write()
} }
+10 -10
View File
@@ -21,8 +21,8 @@ var _ = Suite(&LocalRepoSuite{})
func (s *LocalRepoSuite) SetUpTest(c *C) { func (s *LocalRepoSuite) SetUpTest(c *C) {
s.db, _ = goleveldb.NewOpenDB(c.MkDir()) s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.list = NewPackageList() s.list = NewPackageList()
_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"}) s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"}) s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.reflist = NewPackageRefListFromPackageList(s.list) s.reflist = NewPackageRefListFromPackageList(s.list)
@@ -31,7 +31,7 @@ func (s *LocalRepoSuite) SetUpTest(c *C) {
} }
func (s *LocalRepoSuite) TearDownTest(c *C) { func (s *LocalRepoSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *LocalRepoSuite) TestString(c *C) { func (s *LocalRepoSuite) TestString(c *C) {
@@ -88,14 +88,14 @@ func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
s.collection = NewLocalRepoCollection(s.db) s.collection = NewLocalRepoCollection(s.db)
s.list = NewPackageList() s.list = NewPackageList()
_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"}) s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"}) s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.reflist = NewPackageRefListFromPackageList(s.list) s.reflist = NewPackageRefListFromPackageList(s.list)
} }
func (s *LocalRepoCollectionSuite) TearDownTest(c *C) { func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *LocalRepoCollectionSuite) TestAddByName(c *C) { func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
@@ -156,7 +156,7 @@ func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) { func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
repo := NewLocalRepo("local1", "Comment 1") repo := NewLocalRepo("local1", "Comment 1")
_ = s.collection.Add(repo) s.collection.Add(repo)
count := 0 count := 0
err := s.collection.ForEach(func(*LocalRepo) error { err := s.collection.ForEach(func(*LocalRepo) error {
@@ -178,10 +178,10 @@ func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
func (s *LocalRepoCollectionSuite) TestDrop(c *C) { func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
repo1 := NewLocalRepo("local1", "Comment 1") repo1 := NewLocalRepo("local1", "Comment 1")
_ = s.collection.Add(repo1) s.collection.Add(repo1)
repo2 := NewLocalRepo("local2", "Comment 2") repo2 := NewLocalRepo("local2", "Comment 2")
_ = s.collection.Add(repo2) s.collection.Add(repo2)
r1, _ := s.collection.ByUUID(repo1.UUID) r1, _ := s.collection.ByUUID(repo1.UUID)
c.Check(r1, Equals, repo1) c.Check(r1, Equals, repo1)
@@ -208,6 +208,6 @@ func (s *LocalRepoCollectionSuite) TestDropNonExisting(c *C) {
_, err := s.collection.ByUUID(repo.UUID) _, err := s.collection.ByUUID(repo.UUID)
c.Check(err, ErrorMatches, "local repo .* not found") c.Check(err, ErrorMatches, "local repo .* not found")
_ = s.collection.Drop(repo) err = s.collection.Drop(repo)
c.Check(s.collection.Drop(repo), ErrorMatches, "local repo not found") c.Check(s.collection.Drop(repo), ErrorMatches, "local repo not found")
} }
+1 -1
View File
@@ -565,7 +565,7 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl
} }
return nil, err return nil, err
} }
defer func() { _ = reader.Close() }() defer reader.Close()
contents, err := GetContentsFromDeb(reader, file.Filename) contents, err := GetContentsFromDeb(reader, file.Filename)
if err != nil { if err != nil {
+2 -2
View File
@@ -309,7 +309,7 @@ func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList)
} }
if q.Matches(pkg) { if q.Matches(pkg) {
_ = result.Add(pkg) result.Add(pkg)
} }
} }
@@ -337,7 +337,7 @@ func (collection *PackageCollection) SearchByKey(arch, name, version string) (re
} }
if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version { if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
_ = result.Add(pkg) result.Add(pkg)
} }
} }
+2 -2
View File
@@ -23,7 +23,7 @@ func (s *PackageCollectionSuite) SetUpTest(c *C) {
} }
func (s *PackageCollectionSuite) TearDownTest(c *C) { func (s *PackageCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *PackageCollectionSuite) TestUpdate(c *C) { func (s *PackageCollectionSuite) TestUpdate(c *C) {
@@ -67,7 +67,7 @@ func (s *PackageCollectionSuite) TestByKey(c *C) {
func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) { func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) {
key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1") key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1")
_ = s.db.Put(key, old0_3Package) s.db.Put(key, old0_3Package)
p, err := s.collection.ByKey(key) p, err := s.collection.ByKey(key)
c.Check(err, IsNil) c.Check(err, IsNil)
+1 -1
View File
@@ -64,7 +64,7 @@ func (files PackageFiles) Hash() uint64 {
for _, f := range files { for _, f := range files {
h.Write([]byte(f.Filename)) h.Write([]byte(f.Filename))
_ = binary.Write(h, binary.BigEndian, f.Checksums.Size) binary.Write(h, binary.BigEndian, f.Checksums.Size)
h.Write([]byte(f.Checksums.MD5)) h.Write([]byte(f.Checksums.MD5))
h.Write([]byte(f.Checksums.SHA1)) h.Write([]byte(f.Checksums.SHA1))
h.Write([]byte(f.Checksums.SHA256)) h.Write([]byte(f.Checksums.SHA256))
+2 -2
View File
@@ -1,7 +1,7 @@
package deb package deb
import ( import (
"os" "io/ioutil"
"path/filepath" "path/filepath"
"github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/aptly"
@@ -39,7 +39,7 @@ func (s *PackageFilesSuite) TestVerify(c *C) {
c.Check(result, Equals, false) c.Check(result, Equals, false)
tmpFilepath := filepath.Join(c.MkDir(), "file") tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil) c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs) s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
+5 -5
View File
@@ -2,7 +2,7 @@ package deb
import ( import (
"bytes" "bytes"
"os" "io/ioutil"
"path/filepath" "path/filepath"
"regexp" "regexp"
@@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
} }
func (s *PackageSuite) TestNewInstallerFromPara(c *C) { func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false) repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
downloader := http.NewFakeDownloader() downloader := http.NewFakeDownloader()
downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs") downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list") downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
@@ -395,7 +395,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
p := NewPackageFromControlFile(s.stanza) p := NewPackageFromControlFile(s.stanza)
tmpFilepath := filepath.Join(c.MkDir(), "file") tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, nil, 0777), IsNil) c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs) p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
@@ -434,7 +434,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
}) })
tmpFilepath := filepath.Join(c.MkDir(), "file") tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil) c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs) p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
list, err = p.DownloadList(packagePool, cs) list, err = p.DownloadList(packagePool, cs)
@@ -449,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
cs := files.NewMockChecksumStorage() cs := files.NewMockChecksumStorage()
tmpFilepath := filepath.Join(c.MkDir(), "file") tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil) c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs) p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
+12 -79
View File
@@ -9,7 +9,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
"strconv"
"strings" "strings"
"time" "time"
@@ -55,7 +54,6 @@ type PublishedRepo struct {
Label string Label string
Suite string Suite string
Codename string Codename string
Version string
// Architectures is a list of all architectures published // Architectures is a list of all architectures published
Architectures []string Architectures []string
// SourceKind is "local"/"repo" // SourceKind is "local"/"repo"
@@ -83,11 +81,6 @@ type PublishedRepo struct {
// Provide index files per hash also // Provide index files per hash also
AcquireByHash bool AcquireByHash bool
// An optional field containing a comma separated list
// of OpenPGP key fingerprints to be used
// for validating the next Release file
SignedBy string
// Support multiple distributions // Support multiple distributions
MultiDist bool MultiDist bool
@@ -527,7 +520,6 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
"Origin": p.Origin, "Origin": p.Origin,
"Suite": p.Suite, "Suite": p.Suite,
"Codename": p.Codename, "Codename": p.Codename,
"Version": p.Version,
"NotAutomatic": p.NotAutomatic, "NotAutomatic": p.NotAutomatic,
"ButAutomaticUpgrades": p.ButAutomaticUpgrades, "ButAutomaticUpgrades": p.ButAutomaticUpgrades,
"Prefix": p.Prefix, "Prefix": p.Prefix,
@@ -537,7 +529,6 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
"Storage": p.Storage, "Storage": p.Storage,
"SkipContents": p.SkipContents, "SkipContents": p.SkipContents,
"AcquireByHash": p.AcquireByHash, "AcquireByHash": p.AcquireByHash,
"SignedBy": p.SignedBy,
"MultiDist": p.MultiDist, "MultiDist": p.MultiDist,
}) })
} }
@@ -640,7 +631,7 @@ func (p *PublishedRepo) Components() []string {
return result return result
} }
// SourceNames returns sorted list of published repo source names // Components returns sorted list of published repo source names
func (p *PublishedRepo) SourceNames() []string { func (p *PublishedRepo) SourceNames() []string {
var sources = []string{} var sources = []string{}
@@ -711,7 +702,7 @@ func (p *PublishedRepo) Encode() []byte {
var buf bytes.Buffer var buf bytes.Buffer
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
_ = encoder.Encode(p) encoder.Encode(p)
return buf.Bytes() return buf.Bytes()
} }
@@ -893,7 +884,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
if err != nil { if err != nil {
return err return err
} }
defer func() { _ = os.RemoveAll(tempDir) }() defer os.RemoveAll(tempDir)
indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash, p.SkipBz2) indexes := newIndexFiles(publishedStorage, basePath, tempDir, suffix, p.AcquireByHash, p.SkipBz2)
@@ -979,7 +970,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
contentIndexesMap[key] = contentIndex contentIndexesMap[key] = contentIndex
} }
_ = contentIndex.Push(qualifiedName, contents, batch) contentIndex.Push(qualifiedName, contents, batch)
} }
} }
@@ -1055,38 +1046,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
} }
} }
// Pass-through AppStream (DEP-11) files from snapshots
for component, item := range p.sourceItems {
if item.snapshot == nil || len(item.snapshot.AppStreamFiles) == 0 {
continue
}
prefix := component + "/"
for relPath, poolPath := range item.snapshot.AppStreamFiles {
if !strings.HasPrefix(relPath, prefix) {
continue
}
withinComponent := strings.TrimPrefix(relPath, prefix)
poolFile, err := packagePool.Open(poolPath)
if err != nil {
return fmt.Errorf("unable to open AppStream file from pool: %v", err)
}
bufWriter, err := indexes.SkelIndex(component, withinComponent).BufWriter()
if err != nil {
_ = poolFile.Close()
return fmt.Errorf("unable to generate AppStream index: %v", err)
}
_, err = bufio.NewReader(poolFile).WriteTo(bufWriter)
_ = poolFile.Close()
if err != nil {
return fmt.Errorf("unable to write AppStream file: %v", err)
}
}
}
udebs := []bool{false} udebs := []bool{false}
if hadUdebs { if hadUdebs {
udebs = append(udebs, true) udebs = append(udebs, true)
@@ -1111,12 +1070,6 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
if p.AcquireByHash { if p.AcquireByHash {
release["Acquire-By-Hash"] = "yes" release["Acquire-By-Hash"] = "yes"
} }
if p.SignedBy != "" {
release["Signed-By"] = p.SignedBy
}
if p.Version != "" {
release["Version"] = p.Version
}
var bufWriter *bufio.Writer var bufWriter *bufio.Writer
bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter() bufWriter, err = indexes.ReleaseIndex(component, arch, udeb).BufWriter()
@@ -1173,31 +1126,11 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
release["Label"] = p.GetLabel() release["Label"] = p.GetLabel()
release["Suite"] = p.GetSuite() release["Suite"] = p.GetSuite()
release["Codename"] = p.GetCodename() release["Codename"] = p.GetCodename()
datetimeformat := "Mon, 2 Jan 2006 15:04:05 MST" release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
publishDate := time.Now().UTC()
if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" {
if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil {
publishDate = time.Unix(sec, 0).UTC()
}
}
release["Date"] = publishDate.Format(datetimeformat)
release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ") release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
if p.AcquireByHash { if p.AcquireByHash {
release["Acquire-By-Hash"] = "yes" release["Acquire-By-Hash"] = "yes"
} }
if p.SignedBy != "" {
// "If the field is present, a client should only accept future updates
// to the repository that are signed with keys listed in the field.
// The field should be ignored if the Valid-Until field
// is not present or if it is expired."
release["Signed-By"] = p.SignedBy
// Let's use a century as a "forever" value.
release["Valid-Until"] = publishDate.AddDate(100, 0, 0).Format(datetimeformat)
}
if p.Version != "" {
release["Version"] = p.Version
}
release["Description"] = " Generated by aptly\n" release["Description"] = " Generated by aptly\n"
release["MD5Sum"] = "" release["MD5Sum"] = ""
release["SHA1"] = "" release["SHA1"] = ""
@@ -1342,11 +1275,11 @@ func (collection *PublishedRepoCollection) CheckDuplicate(repo *PublishedRepo) *
// Update stores updated information about repo in DB // Update stores updated information about repo in DB
func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error { func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
batch := collection.db.CreateBatch() batch := collection.db.CreateBatch()
_ = batch.Put(repo.Key(), repo.Encode()) batch.Put(repo.Key(), repo.Encode())
if repo.SourceKind == SourceLocalRepo { if repo.SourceKind == SourceLocalRepo {
for component, item := range repo.sourceItems { for component, item := range repo.sourceItems {
_ = batch.Put(repo.RefKey(component), item.packageRefs.Encode()) batch.Put(repo.RefKey(component), item.packageRefs.Encode())
} }
} }
return batch.Write() return batch.Write()
@@ -1391,7 +1324,7 @@ func (collection *PublishedRepoCollection) LoadShallow(repo *PublishedRepo, coll
// LoadComplete loads complete information on the sources of the repo *and* their packages // LoadComplete loads complete information on the sources of the repo *and* their packages
func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) { func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
_ = collection.LoadShallow(repo, collectionFactory) collection.LoadShallow(repo, collectionFactory)
if repo.SourceKind == SourceSnapshot { if repo.SourceKind == SourceSnapshot {
for _, item := range repo.sourceItems { for _, item := range repo.sourceItems {
@@ -1569,7 +1502,7 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix
return nil, err return nil, err
} }
_ = packageList.ForEach(func(p *Package) error { packageList.ForEach(func(p *Package) error {
poolDir, err := p.PoolDirectory() poolDir, err := p.PoolDirectory()
if err != nil { if err != nil {
return err return err
@@ -1642,7 +1575,7 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published
return err return err
} }
_ = packageList.ForEach(func(p *Package) error { packageList.ForEach(func(p *Package) error {
poolDir, err := p.PoolDirectory() poolDir, err := p.PoolDirectory()
if err != nil { if err != nil {
return err return err
@@ -1776,10 +1709,10 @@ func (collection *PublishedRepoCollection) Remove(publishedStorageProvider aptly
} }
batch := collection.db.CreateBatch() batch := collection.db.CreateBatch()
_ = batch.Delete(repo.Key()) batch.Delete(repo.Key())
for _, component := range repo.Components() { for _, component := range repo.Components() {
_ = batch.Delete(repo.RefKey(component)) batch.Delete(repo.RefKey(component))
} }
return batch.Write() return batch.Write()
+7 -7
View File
@@ -19,13 +19,13 @@ func BenchmarkListReferencedFiles(b *testing.B) {
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
defer func() { _ = os.RemoveAll(tmpDir) }() defer os.RemoveAll(tmpDir)
db, err := goleveldb.NewOpenDB(tmpDir) db, err := goleveldb.NewOpenDB(tmpDir)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
defer func() { _ = db.Close() }() defer db.Close()
factory := NewCollectionFactory(db) factory := NewCollectionFactory(db)
packageCollection := factory.PackageCollection() packageCollection := factory.PackageCollection()
@@ -49,7 +49,7 @@ func BenchmarkListReferencedFiles(b *testing.B) {
Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex), Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex),
}}) }})
_ = packageCollection.UpdateInTransaction(p, transaction) packageCollection.UpdateInTransaction(p, transaction)
sharedRefs.Refs = append(sharedRefs.Refs, p.Key("")) sharedRefs.Refs = append(sharedRefs.Refs, p.Key(""))
} }
@@ -78,7 +78,7 @@ func BenchmarkListReferencedFiles(b *testing.B) {
Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex), Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex),
}}) }})
_ = packageCollection.UpdateInTransaction(p, transaction) packageCollection.UpdateInTransaction(p, transaction)
refs.Refs = append(refs.Refs, p.Key("")) refs.Refs = append(refs.Refs, p.Key(""))
} }
@@ -92,16 +92,16 @@ func BenchmarkListReferencedFiles(b *testing.B) {
repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex) repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex)
repo.DefaultComponent = defaultComponent repo.DefaultComponent = defaultComponent
repo.UpdateRefList(refs.Merge(sharedRefs, false, true)) repo.UpdateRefList(refs.Merge(sharedRefs, false, true))
_ = repoCollection.Add(repo) repoCollection.Add(repo)
publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false) publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
_ = publishCollection.Add(publish) publishCollection.Add(publish)
} }
_ = db.CompactDB() db.CompactDB()
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
+47 -163
View File
@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil"
"os" "os"
"path/filepath" "path/filepath"
"sort" "sort"
@@ -13,7 +14,6 @@ import (
"github.com/aptly-dev/aptly/database" "github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb" "github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
"github.com/ugorji/go/codec" "github.com/ugorji/go/codec"
. "gopkg.in/check.v1" . "gopkg.in/check.v1"
@@ -51,11 +51,11 @@ func (n *NullSigner) SetPassphrase(passphrase, passphraseFile string) {
} }
func (n *NullSigner) DetachedSign(source string, destination string) error { func (n *NullSigner) DetachedSign(source string, destination string) error {
return os.WriteFile(destination, []byte{}, 0644) return ioutil.WriteFile(destination, []byte{}, 0644)
} }
func (n *NullSigner) ClearSign(source string, destination string) error { func (n *NullSigner) ClearSign(source string, destination string) error {
return os.WriteFile(destination, []byte{}, 0644) return ioutil.WriteFile(destination, []byte{}, 0644)
} }
type FakeStorageProvider struct { type FakeStorageProvider struct {
@@ -104,7 +104,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.cs = files.NewMockChecksumStorage() s.cs = files.NewMockChecksumStorage()
tmpFilepath := filepath.Join(c.MkDir(), "file") tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, nil, 0777), IsNil) c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
var err error var err error
s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs) s.p1.Files()[0].PoolPath, err = s.packagePool.Import(tmpFilepath, s.p1.Files()[0].Filename, &s.p1.Files()[0].Checksums, false, s.cs)
@@ -116,24 +116,24 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.reflist = NewPackageRefListFromPackageList(s.list) s.reflist = NewPackageRefListFromPackageList(s.list)
repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false, false) repo, _ := NewRemoteRepo("yandex", "http://mirror.yandex.ru/debian/", "squeeze", []string{"main"}, []string{}, false, false, false)
repo.packageRefs = s.reflist repo.packageRefs = s.reflist
_ = s.factory.RemoteRepoCollection().Add(repo) s.factory.RemoteRepoCollection().Add(repo)
s.localRepo = NewLocalRepo("local1", "comment1") s.localRepo = NewLocalRepo("local1", "comment1")
s.localRepo.packageRefs = s.reflist s.localRepo.packageRefs = s.reflist
_ = s.factory.LocalRepoCollection().Add(s.localRepo) s.factory.LocalRepoCollection().Add(s.localRepo)
s.snapshot, _ = NewSnapshotFromRepository("snap", repo) s.snapshot, _ = NewSnapshotFromRepository("snap", repo)
_ = s.factory.SnapshotCollection().Add(s.snapshot) s.factory.SnapshotCollection().Add(s.snapshot)
s.snapshot2, _ = NewSnapshotFromRepository("snap", repo) s.snapshot2, _ = NewSnapshotFromRepository("snap", repo)
_ = s.factory.SnapshotCollection().Add(s.snapshot2) s.factory.SnapshotCollection().Add(s.snapshot2)
s.packageCollection = s.factory.PackageCollection() s.packageCollection = s.factory.PackageCollection()
_ = s.packageCollection.Update(s.p1) s.packageCollection.Update(s.p1)
_ = s.packageCollection.Update(s.p2) s.packageCollection.Update(s.p2)
_ = s.packageCollection.Update(s.p3) s.packageCollection.Update(s.p3)
s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false) s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
s.repo.SkipContents = true s.repo.SkipContents = true
@@ -152,7 +152,7 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
} }
func (s *PublishedRepoSuite) TearDownTest(c *C) { func (s *PublishedRepoSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) { func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
@@ -179,12 +179,12 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
c.Check(s.repo3.RefList("main").Len(), Equals, 3) c.Check(s.repo3.RefList("main").Len(), Equals, 3)
c.Check(s.repo3.RefList("contrib").Len(), Equals, 3) c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
c.Check(func() { _, _ = NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources") c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources")
c.Check(func() { c.Check(func() {
_, _ = NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false) NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
}, PanicMatches, "sources and components should be equal in size") }, PanicMatches, "sources and components should be equal in size")
c.Check(func() { c.Check(func() {
_, _ = NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false) NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false)
}, PanicMatches, "interface conversion:.*") }, PanicMatches, "interface conversion:.*")
_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false) _, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
@@ -337,7 +337,7 @@ func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
s.localRepo.DefaultDistribution = "precise" s.localRepo.DefaultDistribution = "precise"
s.localRepo.DefaultComponent = "contrib" s.localRepo.DefaultComponent = "contrib"
_ = s.factory.LocalRepoCollection().Update(s.localRepo) s.factory.LocalRepoCollection().Update(s.localRepo)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false) repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
c.Check(err, IsNil) c.Check(err, IsNil)
@@ -426,81 +426,6 @@ func (s *PublishedRepoSuite) TestPublish(c *C) {
c.Assert(err, IsNil) c.Assert(err, IsNil)
} }
func (s *PublishedRepoSuite) TestPublishAppStream(c *C) {
// Components + icons
content1 := []byte("DEP-11 test content for Components-amd64.yml.gz")
tmpFile1 := filepath.Join(c.MkDir(), "Components-amd64.yml.gz")
c.Assert(os.WriteFile(tmpFile1, content1, 0644), IsNil)
checksums1 := utils.ChecksumInfo{Size: int64(len(content1))}
poolPath1, err := s.packagePool.Import(tmpFile1, "Components-amd64.yml.gz", &checksums1, false, s.cs)
c.Assert(err, IsNil)
content2 := []byte("DEP-11 icons tar data")
tmpFile2 := filepath.Join(c.MkDir(), "icons-48x48.tar.gz")
c.Assert(os.WriteFile(tmpFile2, content2, 0644), IsNil)
checksums2 := utils.ChecksumInfo{Size: int64(len(content2))}
poolPath2, err := s.packagePool.Import(tmpFile2, "icons-48x48.tar.gz", &checksums2, false, s.cs)
c.Assert(err, IsNil)
// Include contrib file that should be skipped
contribContent := []byte("DEP-11 contrib content")
tmpFile3 := filepath.Join(c.MkDir(), "Components-contrib.yml.gz")
c.Assert(os.WriteFile(tmpFile3, contribContent, 0644), IsNil)
checksums3 := utils.ChecksumInfo{Size: int64(len(contribContent))}
poolPath3, err := s.packagePool.Import(tmpFile3, "Components-contrib.yml.gz", &checksums3, false, s.cs)
c.Assert(err, IsNil)
s.snapshot.AppStreamFiles = map[string]string{
"main/dep11/Components-amd64.yml.gz": poolPath1,
"main/dep11/icons-48x48.tar.gz": poolPath2,
"contrib/dep11/Components-amd64.yml.gz": poolPath3,
}
err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
c.Assert(err, IsNil)
// Both main files should exist
appstreamPath1 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/Components-amd64.yml.gz")
c.Check(appstreamPath1, PathExists)
actual1, err := os.ReadFile(appstreamPath1)
c.Assert(err, IsNil)
c.Check(actual1, DeepEquals, content1)
appstreamPath2 := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/dep11/icons-48x48.tar.gz")
c.Check(appstreamPath2, PathExists)
actual2, err := os.ReadFile(appstreamPath2)
c.Assert(err, IsNil)
c.Check(actual2, DeepEquals, content2)
// Contrib file should not appear
contribPath := filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/contrib/dep11/Components-amd64.yml.gz")
_, statErr := os.Stat(contribPath)
c.Check(os.IsNotExist(statErr), Equals, true)
// Release file should reference AppStream files
rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
defer func() { _ = rf.Close() }()
cfr := NewControlFileReader(rf, true, false)
st, err := cfr.ReadStanza()
c.Assert(err, IsNil)
c.Check(st["MD5Sum"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
c.Check(st["SHA256"], Matches, "(?s).*main/dep11/Components-amd64\\.yml\\.gz.*")
// Pool open error
s.snapshot.AppStreamFiles = map[string]string{
"main/dep11/Components-amd64.yml.gz": "nonexistent/pool/path",
}
err = s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
c.Assert(err, ErrorMatches, "unable to open AppStream file from pool.*")
}
func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") err := s.repo.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
c.Assert(err, IsNil) c.Assert(err, IsNil)
@@ -509,47 +434,6 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) {
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists)
} }
func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) {
// Test with SOURCE_DATE_EPOCH set
_ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890")
defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "")
c.Assert(err, IsNil)
rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
defer func() { _ = rf.Close() }()
cfr := NewControlFileReader(rf, true, false)
st, err := cfr.ReadStanza()
c.Assert(err, IsNil)
// Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC
c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC")
}
func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) {
// Test with invalid SOURCE_DATE_EPOCH (should fallback to current time)
_ = os.Setenv("SOURCE_DATE_EPOCH", "invalid")
defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }()
err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
c.Assert(err, IsNil)
rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release"))
c.Assert(err, IsNil)
defer func() { _ = rf.Close() }()
cfr := NewControlFileReader(rf, true, false)
st, err := cfr.ReadStanza()
c.Assert(err, IsNil)
// Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH)
c.Check(st["Date"], Not(Equals), "")
c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC")
}
func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) {
err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "")
c.Assert(err, IsNil) c.Assert(err, IsNil)
@@ -642,8 +526,8 @@ func (s *PublishedRepoSuite) TestPublishedRepoRevision(c *C) {
bytes, err := json.Marshal(revision) bytes, err := json.Marshal(revision)
c.Assert(err, IsNil) c.Assert(err, IsNil)
jsonExpected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}` json_expected := `{"Sources":[{"Component":"main","Name":"local1"},{"Component":"test1","Name":"snap1"},{"Component":"test2","Name":"snap2"}]}`
c.Assert(string(bytes), Equals, jsonExpected) c.Assert(string(bytes), Equals, json_expected)
c.Assert(s.repo2.DropRevision(), DeepEquals, revision) c.Assert(s.repo2.DropRevision(), DeepEquals, revision)
c.Assert(s.repo2.Revision, IsNil) c.Assert(s.repo2.Revision, IsNil)
@@ -680,11 +564,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
sort.Sort(snap2Refs) sort.Sort(snap2Refs)
s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2") s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2")
_ = s.snapshotCollection.Add(s.snap1) s.snapshotCollection.Add(s.snap1)
_ = s.snapshotCollection.Add(s.snap2) s.snapshotCollection.Add(s.snap2)
s.localRepo = NewLocalRepo("local1", "comment1") s.localRepo = NewLocalRepo("local1", "comment1")
_ = s.factory.LocalRepoCollection().Add(s.localRepo) s.factory.LocalRepoCollection().Add(s.localRepo)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false) s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false) s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false)
@@ -696,7 +580,7 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
} }
func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) { func (s *PublishedRepoCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) { func (s *PublishedRepoCollectionSuite) TestAddByStoragePrefixDistribution(c *C) {
@@ -793,7 +677,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
var buf bytes.Buffer var buf bytes.Buffer
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
_ = encoder.Encode(&old) encoder.Encode(&old)
c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil) c.Assert(s.db.Put(s.repo1.Key(), buf.Bytes()), IsNil)
c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil) c.Assert(s.db.Put(s.repo1.RefKey(""), s.localRepo.RefList().Encode()), IsNil)
@@ -811,7 +695,7 @@ func (s *PublishedRepoCollectionSuite) TestLoadPre0_6(c *C) {
} }
func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) { func (s *PublishedRepoCollectionSuite) TestForEachAndLen(c *C) {
_ = s.collection.Add(s.repo1) s.collection.Add(s.repo1)
count := 0 count := 0
err := s.collection.ForEach(func(*PublishedRepo) error { err := s.collection.ForEach(func(*PublishedRepo) error {
@@ -871,7 +755,7 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
}) })
snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3") snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
_ = s.snapshotCollection.Add(snap3) s.snapshotCollection.Add(snap3)
// Ensure that adding a second publish point with matching files doesn't give duplicate results. // Ensure that adding a second publish point with matching files doesn't give duplicate results.
repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false) repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
@@ -915,7 +799,7 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1") s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
_ = s.snapshotCollection.Add(s.snap1) s.snapshotCollection.Add(s.snap1)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false) s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false) s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
@@ -924,26 +808,26 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false) s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
s.collection = s.factory.PublishedRepoCollection() s.collection = s.factory.PublishedRepoCollection()
_ = s.collection.Add(s.repo1) s.collection.Add(s.repo1)
_ = s.collection.Add(s.repo2) s.collection.Add(s.repo2)
_ = s.collection.Add(s.repo3) s.collection.Add(s.repo3)
_ = s.collection.Add(s.repo4) s.collection.Add(s.repo4)
_ = s.collection.Add(s.repo5) s.collection.Add(s.repo5)
s.root = c.MkDir() s.root = c.MkDir()
s.publishedStorage = files.NewPublishedStorage(s.root, "", "") s.publishedStorage = files.NewPublishedStorage(s.root, "", "")
_ = s.publishedStorage.MkDir("ppa/dists/anaconda") s.publishedStorage.MkDir("ppa/dists/anaconda")
_ = s.publishedStorage.MkDir("ppa/dists/meduza") s.publishedStorage.MkDir("ppa/dists/meduza")
_ = s.publishedStorage.MkDir("ppa/dists/osminog") s.publishedStorage.MkDir("ppa/dists/osminog")
_ = s.publishedStorage.MkDir("ppa/pool/main") s.publishedStorage.MkDir("ppa/pool/main")
_ = s.publishedStorage.MkDir("ppa/pool/contrib") s.publishedStorage.MkDir("ppa/pool/contrib")
_ = s.publishedStorage.MkDir("dists/anaconda") s.publishedStorage.MkDir("dists/anaconda")
_ = s.publishedStorage.MkDir("pool/main") s.publishedStorage.MkDir("pool/main")
s.root2 = c.MkDir() s.root2 = c.MkDir()
s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "") s.publishedStorage2 = files.NewPublishedStorage(s.root2, "", "")
_ = s.publishedStorage2.MkDir("ppa/dists/osminog") s.publishedStorage2.MkDir("ppa/dists/osminog")
_ = s.publishedStorage2.MkDir("ppa/pool/contrib") s.publishedStorage2.MkDir("ppa/pool/contrib")
s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{ s.provider = &FakeStorageProvider{map[string]aptly.PublishedStorage{
"": s.publishedStorage, "": s.publishedStorage,
@@ -951,11 +835,11 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
} }
func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) { func (s *PublishedRepoRemoveSuite) TearDownTest(c *C) {
_ = s.db.Close() s.db.Close()
} }
func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) { func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
_ = s.repo1.RemoveFiles(s.provider, false, []string{}, nil) s.repo1.RemoveFiles(s.provider, false, []string{}, nil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -969,7 +853,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesOnlyDist(c *C) {
} }
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) { func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
_ = s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil) s.repo1.RemoveFiles(s.provider, false, []string{"main"}, nil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -983,7 +867,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPool(c *C) {
} }
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) { func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
_ = s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil) s.repo1.RemoveFiles(s.provider, false, []string{"main", "contrib"}, nil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
@@ -997,7 +881,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithTwoPools(c *C) {
} }
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) { func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
_ = s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil) s.repo1.RemoveFiles(s.provider, true, []string{"main"}, nil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists)) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), Not(PathExists))
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists)) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), Not(PathExists))
@@ -1011,7 +895,7 @@ func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefix(c *C) {
} }
func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) { func (s *PublishedRepoRemoveSuite) TestRemoveFilesWithPrefixRoot(c *C) {
_ = s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil) s.repo2.RemoveFiles(s.provider, true, []string{"main"}, nil)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/anaconda"), PathExists)
c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists) c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/meduza"), PathExists)
+2 -2
View File
@@ -89,7 +89,7 @@ func (q *OrQuery) Fast(list PackageCatalog) bool {
func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) { func (q *OrQuery) Query(list PackageCatalog) (result *PackageList) {
if q.Fast(list) { if q.Fast(list) {
result = q.L.Query(list) result = q.L.Query(list)
_ = result.Append(q.R.Query(list)) result.Append(q.R.Query(list))
} else { } else {
result = list.Scan(q) result = list.Scan(q)
} }
@@ -245,7 +245,7 @@ func (q *DependencyQuery) Query(list PackageCatalog) (result *PackageList) {
if q.Fast(list) { if q.Fast(list) {
result = NewPackageList() result = NewPackageList()
for _, pkg := range list.Search(q.Dep, true, true) { for _, pkg := range list.Search(q.Dep, true, true) {
_ = result.Add(pkg) result.Add(pkg)
} }
} else { } else {
result = list.Scan(q) result = list.Scan(q)
+2 -2
View File
@@ -54,7 +54,7 @@ func (l *PackageRefList) Swap(i, j int) {
l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i] l.Refs[i], l.Refs[j] = l.Refs[j], l.Refs[i]
} }
// Less compares two refs in lexographical order // Compare compares two refs in lexographical order
func (l *PackageRefList) Less(i, j int) bool { func (l *PackageRefList) Less(i, j int) bool {
return bytes.Compare(l.Refs[i], l.Refs[j]) < 0 return bytes.Compare(l.Refs[i], l.Refs[j]) < 0
} }
@@ -64,7 +64,7 @@ func (l *PackageRefList) Encode() []byte {
var buf bytes.Buffer var buf bytes.Buffer
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{}) encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
_ = encoder.Encode(l) encoder.Encode(l)
return buf.Bytes() return buf.Bytes()
} }

Some files were not shown because too many files have changed in this diff Show More