Compare commits

..

3 Commits

Author SHA1 Message Date
André Roth 31aeb0e334 fix golangci-lint error 2025-02-15 23:48:37 +01:00
hudeng c4323cd588 fix: Add ssdb environment preparation
Change-Id: I8534e66786021ce4384c92a2a8d14aa50839a4da
2025-02-15 23:48:36 +01:00
hudeng b3b6ce3539 feat: database backend add ssdb support
Change-Id: I054c5fc9b02f613601781de8613d684faa0ea7f2
2025-02-15 23:48:36 +01:00
281 changed files with 2341 additions and 7450 deletions
-5
View File
@@ -4,10 +4,6 @@ Fixes #
All new code should be covered with tests, documentation should be updated. CI should pass.
Also, to speed up things, if you could kindly "Allow edits and access to secrets by maintainers" in the
PR settings, as this allows us to rebase the PR on master, fix conflicts, run coverage and help with
implementing code and tests.
## Description of the Change
<!--
@@ -18,7 +14,6 @@ Why this change is important?
## Checklist
- [ ] allow Maintainers to edit PR (rebase, run coverage, help with tests, ...)
- [ ] unit-test added (if change is algorithm)
- [ ] functional test added/updated (if change is functional)
- [ ] man page updated (if applicable)
+28 -102
View File
@@ -17,32 +17,8 @@ env:
DEBIAN_FRONTEND: noninteractive
jobs:
unit-test:
name: "Unit Tests"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Docker Image"
run: |
make docker-image
- name: "Unit Tests"
run: |
make docker-unit-tests
mkdir -p out/coverage
mv unit.out out/coverage/
- uses: actions/upload-artifact@v4
with:
name: unit-tests-coverage
path: out/
test:
name: "System Test"
name: "Test (Ubuntu 22.04)"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
@@ -54,10 +30,10 @@ jobs:
GOPROXY: "https://proxy.golang.org"
steps:
- name: "Install Test Packages"
- name: "Install Packages"
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime
sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
- name: "Checkout Repository"
uses: actions/checkout@v4
@@ -87,10 +63,21 @@ jobs:
with:
directory: ${{ runner.temp }}
- name: "Run Unit Tests"
env:
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make test
- name: "Run Benchmark"
run: |
mkdir -p out/coverage
COVERAGE_DIR=$PWD/out/coverage make bench
COVERAGE_DIR=${{ runner.temp }} make bench
- name: "Run System Tests"
env:
@@ -102,63 +89,30 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
sudo mkdir -p /srv ; sudo chown runner /srv
mkdir -p out/coverage
COVERAGE_DIR=$PWD/out/coverage make system-test
- uses: actions/upload-artifact@v4
with:
name: system-tests-coverage
path: out/
coverage:
name: "Upload Coverage"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
needs:
- unit-test
- test
steps:
- name: "Checkout Repository"
uses: actions/checkout@v4
- name: "Download Unit Test Coverage"
uses: actions/download-artifact@v4
with:
name: unit-tests-coverage
- name: "Download System Test Coverage"
uses: actions/download-artifact@v4
with:
name: system-tests-coverage
COVERAGE_DIR=${{ runner.temp }} make system-test
- name: "Merge Code Coverage"
run: |
# go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad
# ~/go/bin/gocovmerge coverage/*.out > coverage.txt
awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt
go install github.com/wadey/gocovmerge@latest
~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
- name: "Upload Code Coverage"
if: github.actor != 'dependabot[bot]'
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@v2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage.txt
fail_ci_if_error: true
ci-debian-build:
name: "Build"
needs:
- coverage
needs: test
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
name: ["Debian 13/trixie", "Debian 12/bookworm", "Debian 11/bullseye", "Ubuntu 26.04", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
arch: ["amd64", "i386" , "arm64" , "armhf"]
include:
- name: "Debian 13/trixie"
- name: "Debian 13/testing"
suite: trixie
image: debian:trixie-slim
- name: "Debian 12/bookworm"
@@ -167,9 +121,9 @@ jobs:
- name: "Debian 11/bullseye"
suite: bullseye
image: debian:bullseye-slim
- name: "Ubuntu 26.04"
suite: resolute
image: ubuntu:26.04
- name: "Debian 10/buster"
suite: buster
image: debian:buster-slim
- name: "Ubuntu 24.04"
suite: noble
image: ubuntu:24.04
@@ -181,12 +135,11 @@ jobs:
image: ubuntu:20.04
container:
image: ${{ matrix.image }}
options: --user root
env:
APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive
steps:
- name: "Install Build Packages"
- name: "Install packages"
run: |
apt-get update
apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang jq bash-completion lintian \
@@ -254,27 +207,9 @@ jobs:
run: |
.github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
- name: "Get aptly version"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
aptlyver=$(make -s version)
echo "Aptly Version: $aptlyver"
echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
id: releaseversion
- name: "Upload CI Artifacts"
if: github.ref != 'refs/heads/master' && !startsWith(github.event.ref, 'refs/tags')
uses: actions/upload-artifact@v4
with:
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.suite }}_${{ matrix.arch }}
path: build/
retention-days: 7
ci-binary-build:
name: "Build"
needs:
- coverage
needs: test
runs-on: ubuntu-latest
strategy:
matrix:
@@ -332,15 +267,6 @@ jobs:
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression
- name: "Upload CI Artifacts"
uses: actions/upload-artifact@v4
if: "!startsWith(github.event.ref, 'refs/tags')"
with:
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression
retention-days: 7
gh-release:
name: "Github Release"
runs-on: ubuntu-latest
+2 -2
View File
@@ -35,7 +35,7 @@ jobs:
- name: Install and initialize swagger
run: |
go install github.com/swaggo/swag/cmd/swag@latest
swag init -q --propertyStrategy pascalcase --markdownFiles docs
swag init -q --markdownFiles docs
shell: sh
- name: golangci-lint
@@ -44,7 +44,7 @@ jobs:
# Require: The version of golangci-lint to use.
# When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
# When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
version: v1.64.5
version: v1.54.1
# Optional: working directory, useful for monorepos
# working-directory: somedir
-2
View File
@@ -26,8 +26,6 @@ _testmain.go
*.test
coverage.txt
coverage.out
coverage.html
*.pyc
+15 -10
View File
@@ -1,11 +1,16 @@
version: "2"
run:
tests: false
linters:
settings:
staticcheck:
checks:
- "all"
- "-QF1004" # could use strings.ReplaceAll instead
- "-QF1012" # Use fmt.Fprintf(...) instead of WriteString(fmt.Sprintf(...))
- "-QF1003" # could use tagged switch
- "-ST1000" # at least one file in a package should have a package comment
- "-QF1001" # could apply De Morgan's law
disable-all: true
enable:
- goconst
- gofmt
- goimports
- govet
- ineffassign
- misspell
- revive
- staticcheck
- vetshadow
-16
View File
@@ -68,19 +68,3 @@ List of contributors, in chronological order:
* Blake Kostner (https://github.com/btkostner)
* Leigh London (https://github.com/leighlondon)
* Gordian Schoenherr (https://github.com/schoenherrg)
* Silke Hofstra (https://github.com/silkeh)
* Itay Porezky (https://github.com/itayporezky)
* Alejandro Guijarro Monerris (https://github.com/alguimodd)
* JupiterRider (https://github.com/JupiterRider)
* Agustin Henze (https://github.com/agustinhenze)
* Tobias Assarsson (https://github.com/daedaluz)
* Yaksh Bariya (https://github.com/thunder-coding)
* Juan Calderon-Perez (https://github.com/gaby)
* Ato Araki (https://github.com/atotto)
* Roman Lebedev (https://github.com/LebedevRI)
* Brian Witt (https://github.com/bwitt)
* Ales Bregar (https://github.com/abregar)
* Tim Foerster (https://github.com/tonobo)
* Zhang Xiao (https://github.com/xzhang1)
* Tom Nguyen (https://github.com/lecafard)
* Philip Cramer (https://github.com/PhilipCramer)
+2 -2
View File
@@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu
### List of Repositories
* [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page
* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/)
* [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide
fixtures for aptly functional tests
@@ -137,7 +137,7 @@ make docker-unit-tests
In order to run aptly system tests, enter the following:
```
make docker-system-test
make docker-system-tests
```
#### Running golangci-lint
+24 -49
View File
@@ -2,32 +2,13 @@ GOPATH=$(shell go env GOPATH)
VERSION=$(shell make -s version)
PYTHON?=python3
BINPATH?=$(GOPATH)/bin
GOLANGCI_LINT_VERSION=v2.0.2 # version supporting go 1.24
GOLANGCI_LINT_VERSION=v1.54.1 # version supporting go 1.19
COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH)
export PODMAN_USERNS = keep-id
DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src
# Setting TZ for certificates
export TZ=UTC
# Unit Tests and some sysmte tests rely on expired certificates, turn back the time
export TEST_FAKETIME := 2025-01-02 03:04:05
# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests
ifeq ($(COVERAGE_SKIP),1)
COVERAGE_ARG_BUILD :=
COVERAGE_ARG_TEST := --coverage-skip
else
COVERAGE_ARG_BUILD := -coverpkg="./..."
COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR)
endif
# export CAPUTRE=1 for regenrating test gold files
ifeq ($(CAPTURE),1)
CAPTURE_ARG := --capture
endif
# Uncomment to update system test gold files
# CAPTURE := "--capture"
help: ## Print this help
@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -69,7 +50,7 @@ swagger-install:
echo "// @version $(VERSION)" >> docs/swagger.conf
azurite-start:
azurite -l /tmp/aptly-azurite > ~/.azurite.log 2>&1 & \
azurite -l /tmp/aptly-azurite & \
echo $$! > ~/.azurite.pid
azurite-stop:
@@ -77,7 +58,7 @@ azurite-stop:
swagger: swagger-install
# Generate swagger docs
@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
@PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf
etcd-install:
# Install etcd
@@ -88,9 +69,9 @@ flake8: ## run flake8 on system test python files
lint: prepare
# Install golangci-lint
@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
@test -f $(BINPATH)/golangci-lint || go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
# Running lint
@NO_COLOR=true PATH=$(BINPATH)/:$(PATH) golangci-lint run --max-issues-per-linter=0 --max-same-issues=0
@PATH=$(BINPATH)/:$(PATH) golangci-lint run
build: prepare swagger ## Build aptly
@@ -103,11 +84,11 @@ install:
# go install -v
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify which tests to run)
test: prepare swagger etcd-install ## Run unit tests
@echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/aptly-etcd-data; system/t13_etcd/start-etcd.sh > /tmp/aptly-etcd-data/etcd.log 2>&1 &
@echo "\e[33m\e[1mRunning go test ...\e[0m"
faketime "$(TEST_FAKETIME)" go test -v ./... -gocheck.v=true -check.f "$(TEST)" -coverprofile=unit.out; echo $$? > .unit-test.ret
go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/aptly-etcd-data/etcd.log
@@ -115,13 +96,13 @@ test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify
system-test: prepare swagger etcd-install ## Run system tests
# build coverage binary
go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli
go test -v -coverpkg="./..." -c -tags testruncli
# Download fixture-db, fixture-pool, etcd.db
if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests
PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST)
PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
@@ -131,8 +112,7 @@ serve: prepare swagger-install ## Run development server (auto recompiling)
test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
cp debian/aptly.conf ~/.aptly.conf
sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf
sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142
dpkg: prepare swagger ## Build debian packages
@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
@@ -182,49 +162,44 @@ binaries: prepare swagger ## Build binary releases (FreeBSD, macOS, Linux gener
docker-image: ## Build aptly-dev docker image
@docker build -f system/Dockerfile . -t aptly-dev
docker-image-no-cache: ## Build aptly-dev docker image (no cache)
@docker build --no-cache -f system/Dockerfile . -t aptly-dev
docker-build: ## Build aptly in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
docker-shell: ## Run aptly and other commands in docker container
@$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
docker-deb: ## Build debian packages in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
docker-unit-tests: ## Run unit tests in docker container (add TEST=regex to specify which tests to run)
$(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \
docker-unit-test: ## Run unit tests in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
test TEST=$(TEST) \
test \
azurite-stop
docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \
azurite-start \
AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \
AZURE_STORAGE_ACCOUNT=devstoreaccount1 \
AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \
AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \
AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \
system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \
system-test TEST=$(TEST) \
azurite-stop
docker-serve: ## Run development server (auto recompiling) on http://localhost:3142
@$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
docker-lint: ## Run golangci-lint in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
docker-man: ## Create man page in docker container
@$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
mem.png: mem.dat mem.gp
gnuplot mem.gp
+6 -4
View File
@@ -63,7 +63,7 @@ Define Release APT sources in ``/etc/apt/sources.list.d/aptly.list``::
deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/release DIST main
Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
Install aptly packages::
@@ -80,7 +80,7 @@ Define CI APT sources in ``/etc/apt/sources.list.d/aptly-ci.list``::
deb [signed-by=/etc/apt/keyrings/aptly.asc] http://repo.aptly.info/ci DIST main
Where DIST is one of: ``bullseye``, ``bookworm``, ``trixie``, ``focal``, ``jammy``, ``noble``
Where DIST is one of: ``buster``, ``bullseye``, ``bookworm``, ``focal``, ``jammy``, ``noble``
Note: same gpg key is used as for the Upstream Debian Packages.
@@ -111,8 +111,10 @@ With configuration management systems:
- `Chef cookbook <https://github.com/hw-cookbooks/aptly>`_ by Aaron Baer
(Heavy Water Operations, LLC)
- `Puppet module <https://github.com/voxpupuli/puppet-aptly>`_ by
Vox Pupuli
- `Puppet module <https://github.com/alphagov/puppet-aptly>`_ by
Government Digital Services
- `Puppet module <https://github.com/tubemogul/puppet-aptly>`_ by
TubeMogul
- `SaltStack Formula <https://github.com/saltstack-formulas/aptly-formula>`_ by
Forrest Alvarez and Brian Jackson
- `Ansible role <https://github.com/aioue/ansible-role-aptly>`_ by Tom Paine
+1 -3
View File
@@ -12,7 +12,5 @@ git push origin v$version master
```
- run swagger locally (`make docker-serve`)
- copy generated docs/swagger.json to https://github.com/aptly-dev/www.aptly.info/tree/master/static/swagger/aptly_1.x.y.json
- add new version to select tag in content/doc/api/swagger.md line 48
- update version in content/download.md
- push commit to master
- releae www.aptly.info
- create release announcement on https://github.com/aptly-dev/aptly/discussions
+17 -21
View File
@@ -41,10 +41,7 @@ type aptlyVersion struct {
// @Success 200 {object} aptlyVersion
// @Router /api/version [get]
func apiVersion(c *gin.Context) {
version := aptlyVersion{
Version: aptly.Version,
}
c.JSON(200, version)
c.JSON(200, gin.H{"Version": aptly.Version})
}
type aptlyStatus struct {
@@ -70,8 +67,7 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) {
return
}
status := aptlyStatus{Status: "Aptly is ready"}
c.JSON(200, status)
c.JSON(200, gin.H{"Status": "Aptly is ready"})
}
}
@@ -169,7 +165,7 @@ func runTaskInBackground(name string, resources []string, proc task.Process) (ta
return nil, err
}
defer func() { _ = releaseDatabaseConnection() }()
defer releaseDatabaseConnection()
return proc(out, detail)
})
}
@@ -178,18 +174,18 @@ func truthy(value interface{}) bool {
if value == nil {
return false
}
switch v := value.(type) {
switch value.(type) {
case string:
switch strings.ToLower(v) {
switch strings.ToLower(value.(string)) {
case "n", "no", "f", "false", "0", "off":
return false
default:
return true
}
case int:
return v != 0
return !(value.(int) == 0)
case bool:
return v
return value.(bool)
}
return true
}
@@ -214,11 +210,11 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
}
// wait for task to finish
_, _ = context.TaskList().WaitForTaskByID(task.ID)
context.TaskList().WaitForTaskByID(task.ID)
retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
err, _ := context.TaskList().GetTaskErrorByID(task.ID)
_, _ = context.TaskList().DeleteTaskByID(task.ID)
context.TaskList().DeleteTaskByID(task.ID)
if err != nil {
AbortWithJSONError(c, retValue.Code, err)
return
@@ -286,11 +282,11 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
// filter packages by version
if c.Request.URL.Query().Get("maximumVersion") == "1" {
list.PrepareIndex()
_ = list.ForEach(func(p *deb.Package) error {
list.ForEach(func(p *deb.Package) error {
versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
if err != nil {
fmt.Println("filter packages by version, query string parse err: ", err)
_ = c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
} else {
tmpList, err := list.Filter(deb.FilterOptions{
Queries: []deb.PackageQuery{versionQ},
@@ -298,15 +294,15 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
if err == nil {
if tmpList.Len() > 0 {
_ = tmpList.ForEach(func(tp *deb.Package) error {
tmpList.ForEach(func(tp *deb.Package) error {
list.Remove(tp)
return nil
})
_ = list.Add(p)
list.Add(p)
}
} else {
fmt.Println("filter packages by version, filter err: ", err)
_ = c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
}
}
@@ -315,7 +311,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
}
if c.Request.URL.Query().Get("format") == "details" {
_ = list.ForEach(func(p *deb.Package) error {
list.ForEach(func(p *deb.Package) error {
result = append(result, p)
return nil
})
@@ -326,7 +322,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
}
}
func AbortWithJSONError(c *gin.Context, code int, err error) {
func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
_ = c.AbortWithError(code, err)
return c.AbortWithError(code, err)
}
+17 -17
View File
@@ -24,14 +24,14 @@ func Test(t *testing.T) {
TestingT(t)
}
type APISuite struct {
type ApiSuite struct {
context *ctx.AptlyContext
flags *flag.FlagSet
configFile *os.File
router http.Handler
}
var _ = Suite(&APISuite{})
var _ = Suite(&ApiSuite{})
func createTestConfig() *os.File {
file, err := os.CreateTemp("", "aptly")
@@ -45,11 +45,11 @@ func createTestConfig() *os.File {
if err != nil {
return nil
}
_, _ = file.Write(jsonString)
file.Write(jsonString)
return file
}
func (s *APISuite) setupContext() error {
func (s *ApiSuite) setupContext() error {
aptly.Version = "testVersion"
file := createTestConfig()
if nil == file {
@@ -75,23 +75,23 @@ func (s *APISuite) setupContext() error {
return nil
}
func (s *APISuite) SetUpSuite(c *C) {
func (s *ApiSuite) SetUpSuite(c *C) {
err := s.setupContext()
c.Assert(err, IsNil)
}
func (s *APISuite) TearDownSuite(c *C) {
_ = os.Remove(s.configFile.Name())
func (s *ApiSuite) TearDownSuite(c *C) {
os.Remove(s.configFile.Name())
s.context.Shutdown()
}
func (s *APISuite) SetUpTest(c *C) {
func (s *ApiSuite) SetUpTest(c *C) {
}
func (s *APISuite) TearDownTest(c *C) {
func (s *ApiSuite) TearDownTest(c *C) {
}
func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*httptest.ResponseRecorder, error) {
w := httptest.NewRecorder()
req, err := http.NewRequest(method, url, body)
if err != nil {
@@ -102,32 +102,32 @@ func (s *APISuite) HTTPRequest(method string, url string, body io.Reader) (*http
return w, nil
}
func (s *APISuite) TestGinRunsInReleaseMode(c *C) {
func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
c.Check(gin.Mode(), Equals, gin.ReleaseMode)
}
func (s *APISuite) TestGetVersion(c *C) {
func (s *ApiSuite) TestGetVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/version", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
}
func (s *APISuite) TestGetReadiness(c *C) {
func (s *ApiSuite) TestGetReadiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/ready", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
}
func (s *APISuite) TestGetHealthiness(c *C) {
func (s *ApiSuite) TestGetHealthiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/healthy", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
}
func (s *APISuite) TestGetMetrics(c *C) {
func (s *ApiSuite) TestGetMetrics(c *C) {
response, err := s.HTTPRequest("GET", "/api/metrics", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
@@ -141,7 +141,7 @@ func (s *APISuite) TestGetMetrics(c *C) {
c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
}
func (s *APISuite) TestRepoCreate(c *C) {
func (s *ApiSuite) TestRepoCreate(c *C) {
body, err := json.Marshal(gin.H{
"Name": "dummy",
})
@@ -150,7 +150,7 @@ func (s *APISuite) TestRepoCreate(c *C) {
c.Assert(err, IsNil)
}
func (s *APISuite) TestTruthy(c *C) {
func (s *ApiSuite) TestTruthy(c *C) {
c.Check(truthy("no"), Equals, false)
c.Check(truthy("n"), Equals, false)
c.Check(truthy("off"), Equals, false)
+3 -3
View File
@@ -21,7 +21,7 @@ import (
// @Success 200 {object} string "Output"
// @Failure 404 {object} Error "Not Found"
// @Router /api/db/cleanup [post]
func apiDBCleanup(c *gin.Context) {
func apiDbCleanup(c *gin.Context) {
resources := []string{string(task.AllResourcesKey)}
maybeRunTaskInBackground(c, "Clean up db", resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
var err error
@@ -109,8 +109,8 @@ func apiDBCleanup(c *gin.Context) {
if toDelete.Len() > 0 {
batch := db.CreateBatch()
_ = toDelete.ForEach(func(ref []byte) error {
_ = collectionFactory.PackageCollection().DeleteByKey(ref, batch)
toDelete.ForEach(func(ref []byte) error {
collectionFactory.PackageCollection().DeleteByKey(ref, batch)
return nil
})
+2 -41
View File
@@ -13,10 +13,6 @@ import (
"github.com/saracen/walker"
)
// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC).
// In production it calls (*os.File).Sync().
var syncFile = func(f *os.File) error { return f.Sync() }
func verifyPath(path string) bool {
path = filepath.Clean(path)
for _, part := range strings.Split(path, string(filepath.Separator)) {
@@ -118,69 +114,34 @@ func apiFilesUpload(c *gin.Context) {
}
stored := []string{}
openFiles := []*os.File{}
// Write all files first
for _, files := range c.Request.MultipartForm.File {
for _, file := range files {
src, err := file.Open()
if err != nil {
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err)
return
}
defer src.Close()
destPath := filepath.Join(path, filepath.Base(file.Filename))
dst, err := os.Create(destPath)
if err != nil {
_ = src.Close()
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err)
return
}
defer dst.Close()
_, err = io.Copy(dst, src)
_ = src.Close()
if err != nil {
_ = dst.Close()
// Close any files we've opened
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, err)
return
}
// Keep file open for batch sync
openFiles = append(openFiles, dst)
stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename)))
}
}
// Sync all files at once to catch ENOSPC errors
for i, dst := range openFiles {
err := syncFile(dst)
if err != nil {
// Close all files
for _, f := range openFiles {
_ = f.Close()
}
AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err))
return
}
}
// Close all files
for _, dst := range openFiles {
_ = dst.Close()
}
apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
c.JSON(200, stored)
}
-476
View File
@@ -1,476 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"github.com/smira/flag"
. "gopkg.in/check.v1"
)
type FilesUploadDiskFullSuite struct {
aptlyContext *ctx.AptlyContext
flags *flag.FlagSet
configFile *os.File
router http.Handler
}
var _ = Suite(&FilesUploadDiskFullSuite{})
func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) {
aptly.Version = "testVersion"
file, err := os.CreateTemp("", "aptly")
c.Assert(err, IsNil)
s.configFile = file
jsonString, err := json.Marshal(gin.H{
"architectures": []string{},
"rootDir": c.MkDir(),
})
c.Assert(err, IsNil)
_, err = file.Write(jsonString)
c.Assert(err, IsNil)
_ = file.Close()
flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
flags.Bool("no-lock", false, "dummy")
flags.Int("db-open-attempts", 3, "dummy")
flags.String("config", s.configFile.Name(), "dummy")
flags.String("architectures", "", "dummy")
s.flags = flags
aptlyContext, err := ctx.NewContext(s.flags)
c.Assert(err, IsNil)
s.aptlyContext = aptlyContext
s.router = Router(aptlyContext)
context = aptlyContext
}
func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) {
if s.configFile != nil {
_ = os.Remove(s.configFile.Name())
}
if s.aptlyContext != nil {
s.aptlyContext.Shutdown()
}
}
func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) {
testContent := []byte("test file content for upload")
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "testfile.txt")
c.Assert(err, IsNil)
_, err = part.Write(testContent)
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt")
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil)
c.Check(content, DeepEquals, testContent)
}
func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) {
testContent := bytes.Repeat([]byte("A"), 10000)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "largefile.bin")
c.Assert(err, IsNil)
_, err = io.Copy(part, bytes.NewReader(testContent))
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir2", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin")
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil)
c.Check(len(content), Equals, len(testContent))
c.Check(content, DeepEquals, testContent)
}
func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) {
testFiles := map[string][]byte{
"file1.txt": []byte("content of file 1"),
"file2.txt": bytes.Repeat([]byte("B"), 5000),
"file3.deb": []byte("debian package content"),
}
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
for filename, content := range testFiles {
part, err := writer.CreateFormFile("file", filename)
c.Assert(err, IsNil)
_, err = part.Write(content)
c.Assert(err, IsNil)
}
err := writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/multitest", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest")
for filename, expectedContent := range testFiles {
uploadedFile := filepath.Join(uploadDir, filename)
content, err := os.ReadFile(uploadedFile)
c.Assert(err, IsNil, Commentf("Failed to read %s", filename))
c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename))
}
}
func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) {
oldSyncFile := syncFile
syncFile = func(f *os.File) error {
if filepath.Base(f.Name()) == "syncfail.txt" {
return syscall.ENOSPC
}
return nil
}
defer func() { syncFile = oldSyncFile }()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part1, err := writer.CreateFormFile("file", "ok.txt")
c.Assert(err, IsNil)
_, err = part1.Write([]byte("ok"))
c.Assert(err, IsNil)
part2, err := writer.CreateFormFile("file", "syncfail.txt")
c.Assert(err, IsNil)
_, err = part2.Write([]byte("will fail on sync"))
c.Assert(err, IsNil)
err = writer.Close()
c.Assert(err, IsNil)
req, err := http.NewRequest("POST", "/api/files/syncfaildir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) {
c.Check(verifyPath("a/b/c"), Equals, true)
c.Check(verifyPath("../x"), Equals, false)
c.Check(verifyPath("./x"), Equals, true)
c.Check(verifyPath(".."), Equals, false)
c.Check(verifyPath("."), Equals, false)
}
func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
}
func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) {
uploadRoot := s.aptlyContext.UploadPath()
c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil)
c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
body := w.Body.String()
c.Check(strings.Contains(body, "d1"), Equals, true)
c.Check(strings.Contains(body, "d2"), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) {
req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 404)
}
func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dir")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files/dir", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
body := w.Body.String()
c.Check(strings.Contains(body, "a.txt"), Equals, true)
c.Check(strings.Contains(body, "b.txt"), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
_, statErr := os.Stat(base)
c.Check(os.IsNotExist(statErr), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel2")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
_, statErr := os.Stat(filepath.Join(base, "a.txt"))
c.Check(os.IsNotExist(statErr), Equals, true)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "todel3")
c.Assert(os.MkdirAll(base, 0777), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
}
func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) {
req, err := http.NewRequest("DELETE", "/api/files/..", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dirx")
c.Assert(os.MkdirAll(base, 0777), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
req, err := http.NewRequest("GET", "/api/files", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 200)
c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]")
}
func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "noperms")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.Chmod(base, 0), IsNil)
defer func() { _ = os.Chmod(base, 0777) }()
req, err := http.NewRequest("GET", "/api/files/noperms", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile")
c.Assert(os.MkdirAll(base, 0777), IsNil)
subdir := filepath.Join(base, "subdir")
c.Assert(os.MkdirAll(subdir, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil)
req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) {
req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart"))
c.Assert(err, IsNil)
req.Header.Set("Content-Type", "multipart/form-data; boundary=missing")
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) {
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/..", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 400)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) {
_ = os.RemoveAll(s.aptlyContext.UploadPath())
c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil)
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/testdir", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) {
// Pre-populate MultipartForm to inject a FileHeader that fails on Open().
form := &multipart.Form{
File: map[string][]*multipart.FileHeader{
"file": {{Filename: "broken.bin"}},
},
}
req, err := http.NewRequest("POST", "/api/files/openfaildir", nil)
c.Assert(err, IsNil)
req.MultipartForm = form
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) {
base := filepath.Join(s.aptlyContext.UploadPath(), "readonly")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.Chmod(base, 0555), IsNil)
defer func() { _ = os.Chmod(base, 0777) }()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("file", "a.txt")
c.Assert(err, IsNil)
_, err = part.Write([]byte("x"))
c.Assert(err, IsNil)
c.Assert(writer.Close(), IsNil)
req, err := http.NewRequest("POST", "/api/files/readonly", body)
c.Assert(err, IsNil)
req.Header.Set("Content-Type", writer.FormDataContentType())
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) {
parent := s.aptlyContext.UploadPath()
base := filepath.Join(parent, "cantremove")
c.Assert(os.MkdirAll(base, 0777), IsNil)
c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil)
c.Assert(os.Chmod(parent, 0555), IsNil)
defer func() { _ = os.Chmod(parent, 0777) }()
req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil)
c.Assert(err, IsNil)
w := httptest.NewRecorder()
s.router.ServeHTTP(w, req)
c.Assert(w.Code, Equals, 500)
}
+10 -219
View File
@@ -1,7 +1,6 @@
package api
import (
"bufio"
"fmt"
"os"
"os/exec"
@@ -13,60 +12,27 @@ import (
"github.com/gin-gonic/gin"
)
type gpgKeyInfo struct {
// 16-character key ID (short form)
KeyID string `json:"KeyID" example:"8B48AD6246925553"`
// Full fingerprint
Fingerprint string `json:"Fingerprint" example:"D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0"`
// Key validity (u=unknown, f=fulltrust, m=marginal, n=never)
Validity string `json:"Validity" example:"u"`
// User ID(s) associated with this key
UserIDs []string `json:"UserIDs" example:"John Doe <john@example.com>"`
// Creation date (Unix timestamp format from gpg)
CreatedAt string `json:"CreatedAt" example:"2023-01-15"`
}
type gpgKeyListResponse struct {
Keys []gpgKeyInfo `json:"Keys"`
}
type gpgAddKeyParams struct {
// Keyserver, when downloading GpgKeyIDs
Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"`
// GpgKeyIDs to download from Keyserver, comma separated list
GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500,8B48AD6246925553"`
// Armored gpg public ket, instead of downloading from keyserver
GpgKeyArmor string `json:"GpgKeyArmor" example:""`
// Keyring for adding the keys (default: trustedkeys.gpg)
Keyring string `json:"Keyring" example:"trustedkeys.gpg"`
// Add ASCII armored gpg public key, do not download from keyserver
GpgKeyArmor string `json:"GpgKeyArmor" example:""`
// Keyserver to download keys provided in `GpgKeyID`
Keyserver string `json:"Keyserver" example:"hkp://keyserver.ubuntu.com:80"`
// Keys do download from `Keyserver`, separated by space
GpgKeyID string `json:"GpgKeyID" example:"EF0F382A1A7B6500 8B48AD6246925553"`
}
type gpgDeleteKeyParams struct {
// Keyring to delete keys from (default: trustedkeys.gpg)
Keyring string `json:"Keyring" example:"trustedkeys.gpg"`
// Key ID or fingerprint to delete
GpgKeyID string `json:"GpgKeyID" example:"8B48AD6246925553"`
}
// @Summary Add GPG Keys
// @Description **Adds GPG keys to aptly keyring**
// @Description
// @Description Add GPG public keys for verifying remote repositories for mirroring.
// @Description
// @Description Keys can be added in two ways:
// @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty)
// @Description * By providing a `Keyserver` and one or more key IDs in `GpgKeyID`, separated by space (leave GpgKeyArmor empty)
// @Description
// @Description Add GPG public keys for veryfing remote repositories for mirroring.
// @Tags Mirrors
// @Consume json
// @Param request body gpgAddKeyParams true "Parameters"
// @Produce json
// @Success 200 {object} string "OK"
// @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/key [post]
// @Failure 404 {object} Error "Not Found"
// @Router /api/gpg [post]
func apiGPGAddKey(c *gin.Context) {
b := gpgAddKeyParams{}
if c.Bind(&b) != nil {
@@ -94,7 +60,7 @@ func apiGPGAddKey(c *gin.Context) {
AbortWithJSONError(c, 400, err)
return
}
defer func() { _ = os.RemoveAll(tempdir) }()
defer os.RemoveAll(tempdir)
keypath := filepath.Join(tempdir, "key")
keyfile, e := os.Create(keypath)
@@ -134,178 +100,3 @@ func apiGPGAddKey(c *gin.Context) {
c.JSON(200, string(out))
}
// @Summary List GPG Keys
// @Description **Lists all GPG keys in aptly keyring**
// @Description
// @Description Returns all public keys currently installed in the aptly GPG keyring.
// @Description
// @Tags Mirrors
// @Param keyring query string false "Keyring file to list keys from (default: trustedkeys.gpg)" example(trustedkeys.gpg)
// @Produce json
// @Success 200 {object} gpgKeyListResponse "OK"
// @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/keys [get]
func apiGPGListKeys(c *gin.Context) {
keyring := c.DefaultQuery("keyring", "trustedkeys.gpg")
keyring = utils.SanitizePath(keyring)
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
AbortWithJSONError(c, 400, err)
return
}
args := []string{
"--no-default-keyring",
"--with-colons",
"--keyring", keyring,
"--list-keys",
}
cmd := exec.Command(gpg, args...)
out, err := cmd.CombinedOutput()
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("failed to list keys: %s", string(out)))
return
}
keys := parseGPGOutput(string(out))
c.JSON(200, gpgKeyListResponse{Keys: keys})
}
// @Summary Delete GPG Key
// @Description **Deletes a GPG key from aptly keyring**
// @Description
// @Description Removes a public key from the aptly GPG keyring. This is useful for removing
// @Description compromised keys or cleaning up obsolete keys.
// @Description
// @Tags Mirrors
// @Consume json
// @Param request body gpgDeleteKeyParams true "Parameters"
// @Produce json
// @Success 200 {object} string "OK"
// @Failure 400 {object} Error "Bad Request"
// @Router /api/gpg/key [delete]
func apiGPGDeleteKey(c *gin.Context) {
b := gpgDeleteKeyParams{}
if c.Bind(&b) != nil {
AbortWithJSONError(c, 400, fmt.Errorf("invalid request body"))
return
}
if len(strings.TrimSpace(b.GpgKeyID)) == 0 {
AbortWithJSONError(c, 400, fmt.Errorf("GpgKeyID is required"))
return
}
b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
// b.Keyring can be an absolute path
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
AbortWithJSONError(c, 400, err)
return
}
args := []string{
"--no-default-keyring",
"--allow-non-selfsigned-uid",
"--batch",
"--yes",
}
keyring := "trustedkeys.gpg"
if len(b.Keyring) > 0 {
keyring = b.Keyring
}
args = append(args, "--keyring", keyring)
args = append(args, "--delete-keys", b.GpgKeyID)
cmd := exec.Command(gpg, args...)
fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
out, err := cmd.CombinedOutput()
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("failed to delete key: %s", string(out)))
return
}
c.JSON(200, string(out))
}
// parseGPGOutput parses the output of `gpg --with-colons --list-keys`
// and returns a structured list of keys
func parseGPGOutput(output string) []gpgKeyInfo {
var keys []gpgKeyInfo
var currentKey *gpgKeyInfo
scanner := bufio.NewScanner(strings.NewReader(output))
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
parts := strings.Split(line, ":")
if len(parts) < 10 {
continue
}
recordType := parts[0]
// pub: public key record
if recordType == "pub" {
// Save previous key if it exists
if currentKey != nil && currentKey.KeyID != "" {
keys = append(keys, *currentKey)
}
// Create new key entry
// Format: pub:trust:length:algo:keyid:created:expires:uidhash:...
keyID := parts[4]
if len(keyID) >= 16 {
keyID = keyID[len(keyID)-16:] // Last 16 chars = short key ID
}
validity := parts[1]
createdAt := parts[5]
currentKey = &gpgKeyInfo{
KeyID: keyID,
Validity: validity,
CreatedAt: createdAt,
UserIDs: []string{},
Fingerprint: "",
}
}
// uid: user ID record
if recordType == "uid" && currentKey != nil {
// Format: uid:trust:created:expires:keyid:uidhash:uidtype:validity:userID:...
if len(parts) >= 10 {
userID := parts[9]
if userID != "" {
currentKey.UserIDs = append(currentKey.UserIDs, userID)
}
}
}
// fpr: fingerprint record
if recordType == "fpr" && currentKey != nil {
// Format: fpr:::::::::fingerprint:
if len(parts) >= 10 {
fingerprint := parts[9]
currentKey.Fingerprint = fingerprint
}
}
}
// Don't forget the last key
if currentKey != nil && currentKey.KeyID != "" {
keys = append(keys, *currentKey)
}
return keys
}
-451
View File
@@ -1,451 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"os"
"path/filepath"
"strings"
. "gopkg.in/check.v1"
)
type GPGSuite struct {
APISuite
}
var _ = Suite(&GPGSuite{})
func (s *GPGSuite) withFakeGPG(c *C, scriptBody string, test func(scriptPath string)) {
tempDir, err := os.MkdirTemp("", "aptly-fake-gpg")
c.Assert(err, IsNil)
defer func() { _ = os.RemoveAll(tempDir) }()
scriptPath := filepath.Join(tempDir, "gpg")
err = os.WriteFile(scriptPath, []byte(scriptBody), 0o755)
c.Assert(err, IsNil)
oldPath := os.Getenv("PATH")
err = os.Setenv("PATH", tempDir+string(os.PathListSeparator)+oldPath)
c.Assert(err, IsNil)
defer func() { _ = os.Setenv("PATH", oldPath) }()
test(scriptPath)
}
func (s *GPGSuite) fakeGPGScript(c *C, listOutput string, deleteOutput string, deleteError string) string {
return "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then\n" +
" echo 'gpg (GnuPG) 2.2.27'\n" +
" exit 0\n" +
"fi\n" +
"args=\"$*\"\n" +
"if printf '%s' \"$args\" | grep -q -- '--list-keys'; then\n" +
" cat <<'EOF'\n" + listOutput + "\nEOF\n" +
" exit 0\n" +
"fi\n" +
"if printf '%s' \"$args\" | grep -q -- '--delete-keys'; then\n" +
" if [ -n \"" + strings.ReplaceAll(deleteError, "\n", "") + "\" ]; then\n" +
" echo '" + strings.ReplaceAll(deleteError, "'", "'\\''") + "'\n" +
" exit 1\n" +
" fi\n" +
" cat <<'EOF'\n" + deleteOutput + "\nEOF\n" +
" exit 0\n" +
"fi\n" +
"echo 'unexpected invocation' >&2\n" +
"exit 1\n"
}
// TestParseGPGOutputEmpty tests parsing of empty GPG output
func (s *GPGSuite) TestParseGPGOutputEmpty(c *C) {
output := ""
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 0)
}
// TestParseGPGOutputSingleKeyMinimal tests parsing a single key with minimal fields
func (s *GPGSuite) TestParseGPGOutputSingleKeyMinimal(c *C) {
// Minimal valid GPG output with one key
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.KeyID, Equals, "8B48AD6246925553")
c.Check(key.Validity, Equals, "u")
c.Check(key.CreatedAt, Equals, "1611864000")
c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
c.Check(key.UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
}
// TestParseGPGOutputMultipleKeys tests parsing multiple keys
func (s *GPGSuite) TestParseGPGOutputMultipleKeys(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
pub:f:2048:1:A1B2C3D4E5F67890:1580592000:1612128000:uidhash:::scESC:::::::23::0:
uid:f::::1580592000::0987654321::Jane Smith <jane@example.com>::::::::::0:
fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 2)
// First key
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
c.Check(keys[0].Validity, Equals, "u")
c.Check(keys[0].UserIDs, DeepEquals, []string{"John Doe <john@example.com>"})
// Second key
c.Check(keys[1].KeyID, Equals, "A1B2C3D4E5F67890")
c.Check(keys[1].Validity, Equals, "f")
c.Check(keys[1].UserIDs, DeepEquals, []string{"Jane Smith <jane@example.com>"})
}
// TestParseGPGOutputMultipleUIDs tests a key with multiple user IDs
func (s *GPGSuite) TestParseGPGOutputMultipleUIDs(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
uid:u::::1611864000::1234567891::John Doe <john.doe@company.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.UserIDs, HasLen, 2)
c.Check(key.UserIDs, DeepEquals, []string{
"John Doe <john@example.com>",
"John Doe <john.doe@company.com>",
})
}
// TestParseGPGOutputMalformedLines tests that malformed lines are skipped
func (s *GPGSuite) TestParseGPGOutputMalformedLines(c *C) {
// Mix of valid and invalid lines (too few fields)
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
invalid:line:with:only:three:fields
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
}
// TestParseGPGOutputEmptyLines tests that empty lines are skipped
func (s *GPGSuite) TestParseGPGOutputEmptyLines(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
}
// TestParseGPGOutputKeyWithoutUID tests a public key without user ID
func (s *GPGSuite) TestParseGPGOutputKeyWithoutUID(c *C) {
// Key without uid record (should still be included)
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
key := keys[0]
c.Check(key.KeyID, Equals, "8B48AD6246925553")
c.Check(key.UserIDs, HasLen, 0)
c.Check(key.Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
}
// TestParseGPGOutputVariousValidity tests different validity values
func (s *GPGSuite) TestParseGPGOutputVariousValidity(c *C) {
output := `pub:u:4096:1:KEY1111111111111:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::Key1::::::::::0:
fpr:::::::::1111111111111111111111111111111111111111:
pub:f:4096:1:KEY2222222222222:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:f::::1611864000::1234567891::Key2::::::::::0:
fpr:::::::::2222222222222222222222222222222222222222:
pub:m:4096:1:KEY3333333333333:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:m::::1611864000::1234567892::Key3::::::::::0:
fpr:::::::::3333333333333333333333333333333333333333:
pub:n:4096:1:KEY4444444444444:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:n::::1611864000::1234567893::Key4::::::::::0:
fpr:::::::::4444444444444444444444444444444444444444:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 4)
validities := []string{"u", "f", "m", "n"}
for i, validity := range validities {
c.Check(keys[i].Validity, Equals, validity)
}
}
// TestParseGPGOutputShortKeyID tests that key IDs are shortened to 16 chars
func (s *GPGSuite) TestParseGPGOutputShortKeyID(c *C) {
// 40-character key ID that should be shortened to last 16 chars
longKeyID := "0123456789ABCDEF0123456789ABCDEF8B48AD62"
output := `pub:u:4096:1:` + longKeyID + `:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should extract the last 16 characters: 89ABCDEF8B48AD62
c.Check(keys[0].KeyID, Equals, "89ABCDEF8B48AD62")
}
// TestParseGPGOutputSpecialCharactersInUID tests user IDs with special characters
func (s *GPGSuite) TestParseGPGOutputSpecialCharactersInUID(c *C) {
// UID with Unicode characters and special formatting
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::J\xc3\xb6hn D\xc3\xb6\xc3\xa9 (D\xc3\xbcss) <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should preserve the encoded special characters
c.Check(keys[0].UserIDs, HasLen, 1)
}
// TestAPIGPGListKeysDefaultKeyring tests the HTTP endpoint with default keyring
func (s *GPGSuite) TestAPIGPGListKeysDefaultKeyring(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, `pub:u:4096:1:8B48AD6246925553:1611864000:::::
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`, "", ""), func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var result gpgKeyListResponse
err = json.NewDecoder(response.Body).Decode(&result)
c.Assert(err, IsNil)
c.Check(result.Keys, HasLen, 1)
c.Check(result.Keys[0].KeyID, Equals, "8B48AD6246925553")
})
}
// TestAPIGPGListKeysWithKeyringParam tests the HTTP endpoint with custom keyring parameter
func (s *GPGSuite) TestAPIGPGListKeysWithKeyringParam(c *C) {
argFile, err := os.CreateTemp("", "aptly-gpg-args")
c.Assert(err, IsNil)
_ = argFile.Close()
defer func() { _ = os.Remove(argFile.Name()) }()
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
"cat <<'EOF'\n" +
"pub:u:4096:1:8B48AD6246925553:1611864000:::::\n" +
"fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:\n" +
"EOF\n" +
"exit 0\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
response, reqErr := s.HTTPRequest("GET", "/api/gpg/keys?keyring=/custom.gpg", nil)
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 200)
argBytes, readErr := os.ReadFile(argFile.Name())
c.Assert(readErr, IsNil)
c.Check(string(argBytes), Matches, `(?s).*--keyring\ncustom\.gpg\n.*`)
})
}
// TestAPIGPGListKeysResponseFormat tests that the response has the correct structure
func (s *GPGSuite) TestAPIGPGListKeysResponseFormat(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, `pub:f:4096:1:A1B2C3D4E5F67890:1611864000:::::
uid:f::::1611864000::1234567890::Jane Smith <jane@example.com>::::::::::0:
fpr:::::::::E9F0A1B2C3D4E5F6A7B8C9D0E1F2A3B4:`, "", ""), func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
var result gpgKeyListResponse
err = json.NewDecoder(response.Body).Decode(&result)
c.Assert(err, IsNil)
c.Assert(result.Keys, HasLen, 1)
c.Check(result.Keys[0].KeyID, Equals, "A1B2C3D4E5F67890")
c.Check(result.Keys[0].Validity, Equals, "f")
c.Check(result.Keys[0].CreatedAt, Equals, "1611864000")
})
}
// TestParseGPGOutputEdgeCaseUIDWithoutFields tests UID record with missing fields
func (s *GPGSuite) TestParseGPGOutputEdgeCaseUIDWithoutFields(c *C) {
// UID record with fewer than 10 fields
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should not have user ID since it's in field 9 and this record is too short
c.Check(keys[0].UserIDs, HasLen, 0)
}
// TestParseGPGOutputFingerprintWithoutCurrentKey tests FPR record appearing before any PUB
func (s *GPGSuite) TestParseGPGOutputFingerprintWithoutCurrentKey(c *C) {
// FPR record without a preceding PUB (should be ignored)
output := `fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should only have one key with the correct fingerprint
c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
}
// TestParseGPGOutputComplexRealWorldExample tests real-world-like GPG output
func (s *GPGSuite) TestParseGPGOutputComplexRealWorldExample(c *C) {
// Real-world GPG output with multiple keys, UIDs, and other record types (sig, sub)
// Note: sub and sig records are skipped as we only care about pub/uid/fpr
realWorldOutput := `tru::1:1611864000:0:3:1:5
pub:u:4096:1:8B48AD6246925553:1611864000:2023-01-15T00:00:00:::::scESC:::::::23::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:
uid:u::::1611864000::1234567890::John Doe <john@example.com>::::::::::0:
uid:u::::1611864100::1234567891::John Doe <john@work.com>::::::::::0:
pub:f:2048:1:1234567890123456:1580592000:2022-12-31T00:00:00::u:::scESC:::::::23::0:
fpr:::::::::F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9:
uid:f::::1580592000::0987654321::Maintainer Key <maint@example.com>::::::::::0:`
keys := parseGPGOutput(realWorldOutput)
c.Check(keys, HasLen, 2)
// First key should have 2 UIDs
c.Check(keys[0].KeyID, Equals, "8B48AD6246925553")
c.Check(keys[0].UserIDs, HasLen, 2)
c.Check(keys[0].Fingerprint, Equals, "D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0")
// Second key should have 1 UID
c.Check(keys[1].KeyID, Equals, "1234567890123456")
c.Check(keys[1].UserIDs, HasLen, 1)
c.Check(keys[1].Fingerprint, Equals, "F4E3D2C1B0A9F8E7D6C5B4A3F2E1D0C9")
}
// TestParseGPGOutputConsecutiveEmptyUIDs tests handling of consecutive empty user ID fields
func (s *GPGSuite) TestParseGPGOutputConsecutiveEmptyUIDs(c *C) {
output := `pub:u:4096:1:8B48AD6246925553:1611864000:1643400000:uidhash:::scESC:::::::23::0:
uid:u::::1611864000::1234567890:::::::::::0:
uid:u::::1611864000::1234567891::John Doe <john@example.com>::::::::::0:
fpr:::::::::D8E8F5A516E7A2C4F3E4B5A6C7D8E9F0:`
keys := parseGPGOutput(output)
c.Check(keys, HasLen, 1)
// Should skip empty UID but include the non-empty one
c.Check(keys[0].UserIDs, HasLen, 1)
c.Check(keys[0].UserIDs[0], Equals, "John Doe <john@example.com>")
}
// TestGPGDeleteKeyParamsValidation tests gpgDeleteKeyParams validation
func (s *GPGSuite) TestGPGDeleteKeyParamsValidation(c *C) {
// This is a unit test that validates parameter structure (no HTTP needed)
params := gpgDeleteKeyParams{
Keyring: "custom.gpg",
GpgKeyID: "8B48AD6246925553",
}
c.Check(params.Keyring, Equals, "custom.gpg")
c.Check(params.GpgKeyID, Equals, "8B48AD6246925553")
}
// TestAPIGPGDeleteKeyMissingKeyID tests delete with missing key ID parameter
func (s *GPGSuite) TestAPIGPGDeleteKeyMissingKeyID(c *C) {
body, err := json.Marshal(map[string]string{
"Keyring": "trustedkeys.gpg",
// GpgKeyID is missing
})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
}
// TestAPIGPGDeleteKeyInvalidJSON tests delete with invalid JSON request
func (s *GPGSuite) TestAPIGPGDeleteKeyInvalidJSON(c *C) {
response, err := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader([]byte("invalid json")))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
}
// TestAPIGPGDeleteKeySuccess tests successful key deletion
func (s *GPGSuite) TestAPIGPGDeleteKeySuccess(c *C) {
argFile, err := os.CreateTemp("", "aptly-gpg-delete-args")
c.Assert(err, IsNil)
_ = argFile.Close()
defer func() { _ = os.Remove(argFile.Name()) }()
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"printf '%s\n' \"$@\" > '" + argFile.Name() + "'\n" +
"if printf '%s' \"$*\" | grep -q -- '--delete-keys'; then\n" +
"echo 'deleted'\n" +
"exit 0\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
body, marshalErr := json.Marshal(gpgDeleteKeyParams{
Keyring: "/trustedkeys.gpg",
GpgKeyID: "8B48AD6246925553",
})
c.Assert(marshalErr, IsNil)
response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, `"deleted\\n"`)
argBytes, readErr := os.ReadFile(argFile.Name())
c.Assert(readErr, IsNil)
argText := string(argBytes)
c.Check(argText, Matches, `(?s).*--batch\n--yes\n.*`)
c.Check(argText, Matches, `(?s).*--keyring\n/trustedkeys\.gpg\n.*`)
c.Check(argText, Matches, `(?s).*--delete-keys\n8B48AD6246925553\n.*`)
})
}
// TestAPIGPGListKeysCommandFailure tests list error propagation from gpg
func (s *GPGSuite) TestAPIGPGListKeysCommandFailure(c *C) {
script := "#!/bin/sh\n" +
"if [ \"$1\" = \"--version\" ]; then echo 'gpg (GnuPG) 2.2.27'; exit 0; fi\n" +
"if printf '%s' \"$*\" | grep -q -- '--list-keys'; then\n" +
"echo 'keyring missing'\n" +
"exit 1\n" +
"fi\n" +
"exit 1\n"
s.withFakeGPG(c, script, func(_ string) {
response, err := s.HTTPRequest("GET", "/api/gpg/keys", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, `(?s).*failed to list keys: keyring missing.*`)
})
}
// TestAPIGPGDeleteKeyCommandFailure tests delete error propagation from gpg
func (s *GPGSuite) TestAPIGPGDeleteKeyCommandFailure(c *C) {
s.withFakeGPG(c, s.fakeGPGScript(c, "", "", "delete failed"), func(_ string) {
body, err := json.Marshal(gpgDeleteKeyParams{
Keyring: "trustedkeys.gpg",
GpgKeyID: "8B48AD6246925553",
})
c.Assert(err, IsNil)
response, reqErr := s.HTTPRequest("DELETE", "/api/gpg/key", bytes.NewReader(body))
c.Assert(reqErr, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, `(?s).*failed to delete key: delete failed.*`)
})
}
+12 -12
View File
@@ -67,17 +67,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader)
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/", nil)
_ = s.logWriter.Close()
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "warn")
@@ -130,17 +130,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader)
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
_ = s.logWriter.Close()
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info")
@@ -153,17 +153,17 @@ func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader)
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/ready", nil)
_ = s.logWriter.Close()
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "error")
@@ -176,17 +176,17 @@ func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
_, _ = io.Copy(&buf, s.logReader)
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
_ = s.logWriter.Close()
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
_ = json.Unmarshal([]byte(capturedOutput), &jsonMap)
json.Unmarshal([]byte(capturedOutput), &jsonMap)
fmt.Println(capturedOutput)
+66 -225
View File
@@ -31,61 +31,22 @@ func getVerifier(keyRings []string) (pgp.Verifier, error) {
return verifier, nil
}
// stringSlicesEqual compares two string slices for equality (order matters)
func stringSlicesEqual(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// uniqueStrings returns a new slice with only unique strings from the input, sorted
func uniqueStrings(input []string) []string {
if len(input) == 0 {
return input
}
seen := make(map[string]struct{}, len(input))
result := make([]string, 0, len(input))
for _, s := range input {
if _, ok := seen[s]; !ok {
seen[s] = struct{}{}
result = append(result, s)
}
}
sort.Strings(result)
return result
}
// @Summary List Mirrors
// @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API.
// @Tags Mirrors
// @Produce json
// @Success 200 {array} remoteRepoResponse
// @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
result := []remoteRepoResponse{}
err := collection.ForEach(func(repo *deb.RemoteRepo) error {
err := collection.LoadComplete(repo)
if err != nil {
return err
}
result = append(result, newRemoteRepoResponse(repo))
result := []*deb.RemoteRepo{}
collection.ForEach(func(repo *deb.RemoteRepo) error {
result = append(result, repo)
return nil
})
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
return
}
c.JSON(200, result)
}
@@ -111,8 +72,6 @@ type mirrorCreateParams struct {
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to mirror AppStream (DEP-11) metadata
DownloadAppStream bool ` json:"DownloadAppStream"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file
@@ -164,7 +123,7 @@ func apiMirrorsCreate(c *gin.Context) {
}
repo, err := deb.NewRemoteRepo(b.Name, b.ArchiveURL, b.Distribution, b.Components, b.Architectures,
b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller, b.DownloadAppStream)
b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
@@ -216,9 +175,9 @@ func apiMirrorsDrop(c *gin.Context) {
name := c.Params.ByName("name")
force := c.Request.URL.Query().Get("force") == "1"
// Phase 1: Pre-task validation (shallow load for 404 check only)
collectionFactory := context.NewCollectionFactory()
mirrorCollection := collectionFactory.RemoteRepoCollection()
snapshotCollection := collectionFactory.SnapshotCollection()
repo, err := mirrorCollection.ByName(name)
if err != nil {
@@ -228,34 +187,21 @@ func apiMirrorsDrop(c *gin.Context) {
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete mirror %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh collections
taskCollectionFactory := context.NewCollectionFactory()
taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection()
taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
// Fresh load after lock acquired
repo, err := taskMirrorCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
}
err = repo.CheckLock()
err := repo.CheckLock()
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
}
if !force {
// Fresh checks with current collections
snapshots := taskSnapshotCollection.ByRemoteRepoSource(repo)
snapshots := snapshotCollection.ByRemoteRepoSource(repo)
if len(snapshots) > 0 {
return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
}
}
err = taskMirrorCollection.Drop(repo)
err = mirrorCollection.Drop(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
}
@@ -286,7 +232,6 @@ func apiMirrorsShow(c *gin.Context) {
err = collection.LoadComplete(repo)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
return
}
c.JSON(200, repo)
@@ -374,7 +319,7 @@ func apiMirrorsPackages(c *gin.Context) {
}
if c.Request.URL.Query().Get("format") == "details" {
_ = list.ForEach(func(p *deb.Package) error {
list.ForEach(func(p *deb.Package) error {
result = append(result, p)
return nil
})
@@ -385,133 +330,29 @@ func apiMirrorsPackages(c *gin.Context) {
}
}
type mirrorEditParams struct {
// Package query that is applied to mirror packages
Filter *string ` json:"Filter" example:"xserver-xorg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps *bool ` json:"FilterWithDeps"`
// Set "true" to mirror installer files
DownloadInstaller *bool `json:"DownloadInstaller"`
// Set "true" to mirror source packages
DownloadSources *bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs *bool ` json:"DownloadUdebs"`
// URL of the archive to mirror
ArchiveURL *string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Comma separated list of architectures
Architectures *[]string `json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file if a mirror update is required.
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures *bool ` json:"IgnoreSignatures"`
}
// @Summary Edit Mirror
// @Description **Edit mirror config**
// @Tags Mirrors
// @Param name path string true "mirror name to edit"
// @Consume json
// @Param request body mirrorEditParams true "Parameters"
// @Produce json
// @Success 200 {object} deb.RemoteRepo "Mirror was edited successfully"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 409 {object} Error "Aptly db locked"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [post]
func apiMirrorsEdit(c *gin.Context) {
var (
err error
b mirrorEditParams
repo *deb.RemoteRepo
)
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
name := c.Params.ByName("name")
repo, err = collection.ByName(name)
if err != nil {
AbortWithJSONError(c, 404, fmt.Errorf("unable to edit: %s", err))
return
}
err = repo.CheckLock()
if err != nil {
AbortWithJSONError(c, 409, fmt.Errorf("unable to edit: %s", err))
return
}
if c.Bind(&b) != nil {
return
}
fetchMirror := false
ignoreSignatures := context.Config().GpgDisableVerify
if b.Filter != nil {
repo.Filter = *b.Filter
}
if b.FilterWithDeps != nil {
repo.FilterWithDeps = *b.FilterWithDeps
}
if b.DownloadInstaller != nil {
repo.DownloadInstaller = *b.DownloadInstaller
}
if b.DownloadSources != nil {
repo.DownloadSources = *b.DownloadSources
}
if b.DownloadUdebs != nil {
repo.DownloadUdebs = *b.DownloadUdebs
}
if b.ArchiveURL != nil && *b.ArchiveURL != repo.ArchiveRoot {
repo.SetArchiveRoot(*b.ArchiveURL)
fetchMirror = true
}
if b.Architectures != nil {
uniqueArchitectures := uniqueStrings(*b.Architectures)
if !stringSlicesEqual(uniqueArchitectures, uniqueStrings(repo.Architectures)) {
repo.Architectures = uniqueArchitectures
fetchMirror = true
}
}
if b.IgnoreSignatures != nil {
ignoreSignatures = *b.IgnoreSignatures
}
if repo.IsFlat() && repo.DownloadUdebs {
AbortWithJSONError(c, 400, fmt.Errorf("unable to edit: flat mirrors don't support udebs"))
return
}
if fetchMirror {
verifier, err := getVerifier(b.Keyrings)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
return
}
}
err = collection.Update(repo)
if err != nil {
AbortWithJSONError(c, 500, fmt.Errorf("unable to edit: %s", err))
return
}
c.JSON(200, repo)
}
type mirrorUpdateParams struct {
// Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"`
// Gpg keyring(s) for verifying Release file
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures
@@ -520,8 +361,6 @@ type mirrorUpdateParams struct {
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
// Set "true" to download only the latest version per package/architecture
LatestOnly bool ` json:"LatestOnly"`
}
// @Summary Update Mirror
@@ -548,14 +387,21 @@ func apiMirrorsUpdate(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
name := c.Params.ByName("name")
remote, err = collection.ByName(name)
remote, err = collection.ByName(c.Params.ByName("name"))
if err != nil {
AbortWithJSONError(c, 404, err)
return
}
b.Name = remote.Name
b.DownloadUdebs = remote.DownloadUdebs
b.DownloadSources = remote.DownloadSources
b.SkipComponentCheck = remote.SkipComponentCheck
b.SkipArchitectureCheck = remote.SkipArchitectureCheck
b.FilterWithDeps = remote.FilterWithDeps
b.Filter = remote.Filter
b.Architectures = remote.Architectures
b.Components = remote.Components
b.IgnoreSignatures = context.Config().GpgDisableVerify
log.Info().Msgf("%s: Starting mirror update", b.Name)
@@ -564,7 +410,6 @@ func apiMirrorsUpdate(c *gin.Context) {
return
}
// Pre-task validation of new name if provided
if b.Name != remote.Name {
_, err = collection.ByName(b.Name)
if err == nil {
@@ -573,6 +418,27 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}
if b.DownloadUdebs != remote.DownloadUdebs {
if remote.IsFlat() && b.DownloadUdebs {
AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
return
}
}
if b.ArchiveURL != "" {
remote.SetArchiveRoot(b.ArchiveURL)
}
remote.Name = b.Name
remote.DownloadUdebs = b.DownloadUdebs
remote.DownloadSources = b.DownloadSources
remote.SkipComponentCheck = b.SkipComponentCheck
remote.SkipArchitectureCheck = b.SkipArchitectureCheck
remote.FilterWithDeps = b.FilterWithDeps
remote.Filter = b.Filter
remote.Architectures = b.Architectures
remote.Components = b.Components
verifier, err := getVerifier(b.Keyrings)
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
@@ -581,26 +447,9 @@ func apiMirrorsUpdate(c *gin.Context) {
resources := []string{string(remote.Key())}
maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh factory
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.RemoteRepoCollection()
// Fresh load after lock acquired (use captured `name` variable, not gin context)
remote, err := taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
// Fresh rename check inside lock (if renaming)
if b.Name != remote.Name {
_, err := taskCollection.ByName(b.Name)
if err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: mirror %s already exists", b.Name)
}
}
downloader := context.NewDownloader(out)
err = remote.Fetch(downloader, verifier, b.IgnoreSignatures)
err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -612,19 +461,11 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck)
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
if remote.DownloadAppStream && !remote.IsFlat() {
err = remote.DownloadAppStreamFiles(out, downloader,
context.PackagePool(), collectionFactory.ChecksumCollection(nil), b.IgnoreChecksums)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
}
if remote.Filter != "" {
var filterQuery deb.PackageQuery
@@ -640,7 +481,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages, b.LatestOnly)
collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -650,7 +491,7 @@ func apiMirrorsUpdate(c *gin.Context) {
e := context.ReOpenDatabase()
if e == nil {
remote.MarkAsIdle()
_ = collection.Update(remote)
collection.Update(remote)
}
}()
@@ -738,7 +579,7 @@ func apiMirrorsUpdate(c *gin.Context) {
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
_ = file.Close()
file.Close()
}
}
if e != nil {
@@ -812,8 +653,8 @@ func apiMirrorsUpdate(c *gin.Context) {
}
log.Info().Msgf("%s: Finalizing download...", b.Name)
_ = remote.FinalizeDownload(taskCollectionFactory, out)
err = taskCollection.Update(remote)
remote.FinalizeDownload(collectionFactory, out)
err = collectionFactory.RemoteRepoCollection().Update(remote)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
+2 -67
View File
@@ -4,13 +4,12 @@ import (
"bytes"
"encoding/json"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
. "gopkg.in/check.v1"
)
type MirrorSuite struct {
APISuite
ApiSuite
}
var _ = Suite(&MirrorSuite{})
@@ -18,10 +17,7 @@ var _ = Suite(&MirrorSuite{})
func (s *MirrorSuite) TestGetMirrors(c *C) {
response, _ := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Check(response.Code, Equals, 200)
var mirrors []map[string]interface{}
err := json.Unmarshal(response.Body.Bytes(), &mirrors)
c.Assert(err, IsNil)
c.Check(response.Body.String(), Equals, "[]")
}
func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
@@ -30,21 +26,6 @@ func (s *MirrorSuite) TestDeleteMirrorNonExisting(c *C) {
c.Check(response.Body.String(), Equals, "{\"error\":\"unable to drop: mirror with name does-not-exist not found\"}")
}
func (s *MirrorSuite) TestCreateMirrorFlatWithAppStream(c *C) {
body, err := json.Marshal(gin.H{
"Name": "test-flat-appstream",
"ArchiveURL": "http://example.com/repo/",
"Distribution": "./",
"DownloadAppStream": true,
})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("POST", "/api/mirrors", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Matches, ".*AppStream.*flat.*")
}
func (s *MirrorSuite) TestCreateMirror(c *C) {
c.ExpectFailure("Need to mock downloads")
body, err := json.Marshal(gin.H{
@@ -57,49 +38,3 @@ func (s *MirrorSuite) TestCreateMirror(c *C) {
c.Check(response.Code, Equals, 400)
c.Check(response.Body.String(), Equals, "")
}
func (s *MirrorSuite) TestGetMirrorsIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().RemoteRepoCollection()
repo, err := deb.NewRemoteRepo("count-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
c.Assert(err, IsNil)
err = collection.Add(repo)
c.Assert(err, IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), makePackageRefList(c).Encode())
response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var mirrors []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &mirrors)
c.Assert(err, IsNil)
found := false
for _, mirror := range mirrors {
if mirror["Name"] == "count-mirror" {
found = true
value, ok := mirror["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *MirrorSuite) TestGetMirrorsReturns500OnCorruptRefList(c *C) {
collection := s.context.NewCollectionFactory().RemoteRepoCollection()
repo, err := deb.NewRemoteRepo("broken-mirror", "http://example.com/debian", "stable", []string{"main"}, []string{}, false, false, false, false)
c.Assert(err, IsNil)
c.Assert(collection.Add(repo), IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
response, err := s.HTTPRequest("GET", "/api/mirrors", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*unable to show:.*")
}
-30
View File
@@ -1,30 +0,0 @@
package api
import "github.com/aptly-dev/aptly/deb"
type remoteRepoResponse struct {
*deb.RemoteRepo
NumPackages int `json:"NumPackages"`
}
type localRepoResponse struct {
*deb.LocalRepo
NumPackages int `json:"NumPackages"`
}
type snapshotResponse struct {
*deb.Snapshot
NumPackages int `json:"NumPackages"`
}
func newRemoteRepoResponse(repo *deb.RemoteRepo) remoteRepoResponse {
return remoteRepoResponse{RemoteRepo: repo, NumPackages: repo.NumPackages()}
}
func newLocalRepoResponse(repo *deb.LocalRepo) localRepoResponse {
return localRepoResponse{LocalRepo: repo, NumPackages: repo.NumPackages()}
}
func newSnapshotResponse(snapshot *deb.Snapshot) snapshotResponse {
return snapshotResponse{Snapshot: snapshot, NumPackages: snapshot.NumPackages()}
}
-19
View File
@@ -1,19 +0,0 @@
package api
import (
"github.com/aptly-dev/aptly/deb"
. "gopkg.in/check.v1"
)
func makePackageRefList(c *C) *deb.PackageRefList {
list := deb.NewPackageList()
c.Assert(list.Add(&deb.Package{Name: "libcount", Version: "1.0", Architecture: "amd64"}), IsNil)
c.Assert(list.Add(&deb.Package{Name: "appcount", Version: "2.0", Architecture: "all"}), IsNil)
return deb.NewPackageRefListFromPackageList(list)
}
func putRawDBValue(c *C, s *APISuite, key []byte, value []byte) {
db, err := s.context.Database()
c.Assert(err, IsNil)
c.Assert(db.Put(key, value), IsNil)
}
+1 -1
View File
@@ -5,7 +5,7 @@ import (
)
type PackagesSuite struct {
APISuite
ApiSuite
}
var _ = Suite(&PackagesSuite{})
+178 -354
View File
@@ -2,7 +2,6 @@ package api
import (
"fmt"
"log"
"net/http"
"strings"
@@ -17,8 +16,8 @@ import (
type signingParams struct {
// Don't sign published repository
Skip bool ` json:"Skip" example:"false"`
// GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used
GpgKey string ` json:"GpgKey" example:"KEY_ID_a, KEY_ID_b"`
// GPG key ID to use when signing the release, if not specified default key is used
GpgKey string ` json:"GpgKey" example:"A0546A43624A8331"`
// GPG keyring to use (instead of default)
Keyring string ` json:"Keyring" example:"trustedkeys.gpg"`
// GPG secret keyring to use (instead of default) Note: depreciated with gpg2
@@ -42,21 +41,7 @@ func getSigner(options *signingParams) (pgp.Signer, error) {
}
signer := context.GetSigner()
var multiGpgKeys []string
// REST params have priority over config
if options.GpgKey != "" {
for _, p := range strings.Split(options.GpgKey, ",") {
if t := strings.TrimSpace(p); t != "" {
multiGpgKeys = append(multiGpgKeys, t)
}
}
} else if len(context.Config().GpgKeys) > 0 {
multiGpgKeys = context.Config().GpgKeys
}
for _, gpgKey := range multiGpgKeys {
signer.SetKey(gpgKey)
}
signer.SetKey(options.GpgKey)
signer.SetKeyRing(options.Keyring, options.SecretKeyring)
signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
@@ -125,7 +110,7 @@ func apiPublishList(c *gin.Context) {
// @Description See also: `aptly publish show`
// @Tags Publish
// @Produce json
// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs"
// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs"
// @Param distribution path string true "distribution name"
// @Success 200 {object} deb.PublishedRepo
// @Failure 404 {object} Error "Published repository not found"
@@ -183,12 +168,8 @@ type publishedRepoCreateParams struct {
SkipBz2 *bool ` json:"SkipBz2" example:"false"`
// Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file.
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"`
// Version of the release
Version string ` json:"Version" example:""`
}
// @Summary Create Published Repository
@@ -299,25 +280,11 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
multiDist = *b.MultiDist
}
// Pre-register the published repo key in resources so that concurrent
// POST requests for the same prefix/distribution are serialized by the
// task queue rather than racing on CheckDuplicate + Add.
if b.Distribution != "" {
storagePrefix := prefix
if storage != "" {
storagePrefix = storage + ":" + prefix
}
resources = append(resources, "U"+storagePrefix+">>"+b.Distribution)
} else {
log.Printf("distribution not specified for publish to prefix '%s' - unable to lock ", prefix)
}
collection := collectionFactory.PublishedRepoCollection()
taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"",
b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`))
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
taskDetail := task.PublishDetail{
Detail: detail,
}
@@ -329,10 +296,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
for _, source := range sources {
switch s := source.(type) {
case *deb.Snapshot:
snapshotCollection := taskCollectionFactory.SnapshotCollection()
snapshotCollection := collectionFactory.SnapshotCollection()
err = snapshotCollection.LoadComplete(s)
case *deb.LocalRepo:
localCollection := taskCollectionFactory.LocalRepoCollection()
localCollection := collectionFactory.LocalRepoCollection()
err = localCollection.LoadComplete(s)
default:
err = fmt.Errorf("unexpected type for source: %T", source)
@@ -342,11 +309,13 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
}
}
published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, taskCollectionFactory, multiDist)
published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
}
resources = append(resources, string(published.Key()))
if b.Origin != "" {
published.Origin = b.Origin
}
@@ -372,26 +341,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
published.AcquireByHash = *b.AcquireByHash
}
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.Version != "" {
published.Version = b.Version
}
duplicate := taskCollection.CheckDuplicate(published)
duplicate := collection.CheckDuplicate(published)
if duplicate != nil {
_ = taskCollectionFactory.PublishedRepoCollection().LoadComplete(duplicate, taskCollectionFactory)
collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
}
err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err)
}
err = taskCollection.Add(published)
err = collection.Add(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -415,16 +376,8 @@ type publishedRepoUpdateSwitchParams struct {
Snapshots []sourceParams ` json:"Snapshots"`
// Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"`
// Value of Label: field in published repository stanza
Label *string ` json:"Label" example:"Debian"`
// Value of Origin: field in published repository stanza
Origin *string ` json:"Origin" example:"Debian"`
// Version of the release: Optional
Version *string ` json:"Version" example:"13.3"`
}
// @Summary Update Published Repository
@@ -440,6 +393,7 @@ type publishedRepoUpdateSwitchParams struct {
// @Description
// @Description See also: `aptly publish update` / `aptly publish switch`
// @Tags Publish
// @Produce json
// @Param prefix path string true "publishing prefix"
// @Param distribution path string true "distribution name"
// @Param _async query bool false "Run in background and return task object"
@@ -471,7 +425,6 @@ func apiPublishUpdateSwitch(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
snapshotCollection := collectionFactory.SnapshotCollection()
localRepoCollection := collectionFactory.LocalRepoCollection()
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
@@ -479,76 +432,46 @@ func apiPublishUpdateSwitch(c *gin.Context) {
return
}
resources := []string{string(published.Key())}
if published.SourceKind == deb.SourceLocalRepo {
if len(b.Snapshots) > 0 {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
return
}
for _, uuid := range published.Sources {
repo, err2 := localRepoCollection.ByUUID(uuid)
if err2 != nil {
AbortWithJSONError(c, http.StatusNotFound, err2)
return
}
resources = append(resources, string(repo.Key()))
}
} else if published.SourceKind == deb.SourceSnapshot {
for _, snapshotInfo := range b.Snapshots {
snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
_, err2 := snapshotCollection.ByName(snapshotInfo.Name)
if err2 != nil {
AbortWithJSONError(c, http.StatusNotFound, err2)
return
}
resources = append(resources, string(snapshot.ResourceKey()))
}
} else {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type"))
return
}
// Field mutations and fresh DB load are deferred to inside the task so
// they always operate on a consistent state after the lock is held.
if b.SkipContents != nil {
published.SkipContents = *b.SkipContents
}
if b.SkipBz2 != nil {
published.SkipBz2 = *b.SkipBz2
}
if b.AcquireByHash != nil {
published.AcquireByHash = *b.AcquireByHash
}
if b.MultiDist != nil {
published.MultiDist = *b.MultiDist
}
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
// Apply field mutations on the freshly loaded object.
if b.SkipContents != nil {
published.SkipContents = *b.SkipContents
}
if b.SkipBz2 != nil {
published.SkipBz2 = *b.SkipBz2
}
if b.AcquireByHash != nil {
published.AcquireByHash = *b.AcquireByHash
}
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.MultiDist != nil {
published.MultiDist = *b.MultiDist
}
if b.Label != nil {
published.Label = *b.Label
}
if b.Origin != nil {
published.Origin = *b.Origin
}
if b.Version != nil {
published.Version = *b.Version
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
}
revision := published.ObtainRevision()
@@ -562,17 +485,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
}
}
result, err := published.Update(taskCollectionFactory, out)
result, err := published.Update(collectionFactory, out)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
}
err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("Unable to update: %s", err)
}
err = taskCollection.Update(published)
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -580,7 +503,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
if b.SkipCleanup == nil || !*b.SkipCleanup {
cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -630,11 +553,8 @@ func apiPublishDrop(c *gin.Context) {
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
err := taskCollection.Remove(context, storage, prefix, distribution,
taskCollectionFactory, out, force, skipCleanup)
err := collection.Remove(context, storage, prefix, distribution,
collectionFactory, out, force, skipCleanup)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err)
}
@@ -670,52 +590,43 @@ func apiPublishAddSource(c *gin.Context) {
storage, prefix := deb.ParsePrefix(param)
distribution := slashEscape(c.Params.ByName("distribution"))
if c.Bind(&b) != nil {
return
}
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly (no LoadComplete) to verify existence and obtain the
// resource key and task name. The actual mutation is performed inside
// the task on a freshly loaded copy to prevent lost-update races.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err))
return
}
if c.Bind(&b) != nil {
return
}
revision := published.ObtainRevision()
sources := revision.Sources
component := b.Component
name := b.Name
_, exists := sources[component]
if exists {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component))
return
}
sources[component] = name
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to create: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
component := b.Component
name := b.Name
_, exists := sources[component]
if exists {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("unable to create: Component '%s' already exists", component)
}
sources[component] = name
err = taskCollection.Update(published)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -797,48 +708,39 @@ func apiPublishSetSources(c *gin.Context) {
storage, prefix := deb.ParsePrefix(param)
distribution := slashEscape(c.Params.ByName("distribution"))
if c.Bind(&b) != nil {
return
}
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly for 404 check, resource key, and task name.
// Full load and mutation happen inside the task.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
return
}
if c.Bind(&b) != nil {
return
}
revision := published.ObtainRevision()
sources := make(map[string]string, len(b))
revision.Sources = sources
for _, source := range b {
component := source.Component
name := source.Name
sources[component] = name
}
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
revision := published.ObtainRevision()
sources := make(map[string]string, len(b))
revision.Sources = sources
for _, source := range b {
component := source.Component
name := source.Name
sources[component] = name
}
err = taskCollection.Update(published)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -871,33 +773,24 @@ func apiPublishDropChanges(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly for 404 check, resource key, and task name.
// Full load and DropRevision happen inside the task.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
return
}
published.DropRevision()
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
}
published.DropRevision()
err = taskCollection.Update(published)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -933,58 +826,51 @@ func apiPublishUpdateSource(c *gin.Context) {
param := slashEscape(c.Params.ByName("prefix"))
storage, prefix := deb.ParsePrefix(param)
distribution := slashEscape(c.Params.ByName("distribution"))
urlComponent := slashEscape(c.Params.ByName("component"))
// Default component to the URL path segment; the body may rename it.
b.Component = urlComponent
if c.Bind(&b) != nil {
return
}
component := slashEscape(c.Params.ByName("component"))
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly for 404 check, resource key, and task name.
// Full load and mutation happen inside the task.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
return
}
revision := published.ObtainRevision()
sources := revision.Sources
_, exists := sources[component]
if !exists {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component))
return
}
b.Component = component
b.Name = revision.Sources[component]
if c.Bind(&b) != nil {
return
}
if b.Component != component {
delete(sources, component)
}
component = b.Component
name := b.Name
sources[component] = name
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
_, exists := sources[urlComponent]
if !exists {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: Component '%s' does not exist", urlComponent)
}
if b.Component != urlComponent {
delete(sources, urlComponent)
}
newComponent := b.Component
name := b.Name
sources[newComponent] = name
err = taskCollection.Update(published)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -1021,41 +907,33 @@ func apiPublishRemoveSource(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly for 404 check, resource key, and task name.
// Full load and mutation happen inside the task.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err))
return
}
revision := published.ObtainRevision()
sources := revision.Sources
_, exists := sources[component]
if !exists {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component))
return
}
delete(sources, component)
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err)
}
revision := published.ObtainRevision()
sources := revision.Sources
_, exists := sources[component]
if !exists {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: Component '%s' does not exist", component)
}
delete(sources, component)
err = taskCollection.Update(published)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -1077,16 +955,8 @@ type publishedRepoUpdateParams struct {
SkipCleanup *bool ` json:"SkipCleanup" example:"false"`
// Provide index files by hash
AcquireByHash *bool ` json:"AcquireByHash" example:"false"`
// An optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file
SignedBy *string ` json:"SignedBy" example:""`
// Enable multiple packages with the same filename in different distributions
MultiDist *bool ` json:"MultiDist" example:"false"`
// Value of Label: field in published repository stanza
Label *string ` json:"Label" example:"Debian"`
// Value of Origin: field in published repository stanza
Origin *string ` json:"Origin" example:"Debian"`
// Version of the release: Optional
Version *string ` json:"Version" example:"13.3"`
}
// @Summary Update Published Repository
@@ -1127,94 +997,48 @@ func apiPublishUpdate(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
// Load shallowly for 404 check, resource key, and task name.
// Full load and field mutations happen inside the task.
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err))
return
}
resources := []string{string(published.Key())}
// Lock source repos / snapshots the same way apiPublishUpdateSwitch does,
// because published.Update() reads from them and concurrent modification
// would produce an inconsistent view.
snapshotCollection := collectionFactory.SnapshotCollection()
localRepoCollection := collectionFactory.LocalRepoCollection()
if published.SourceKind == deb.SourceLocalRepo {
for _, uuid := range published.Sources {
repo, err2 := localRepoCollection.ByUUID(uuid)
if err2 != nil {
AbortWithJSONError(c, http.StatusNotFound, err2)
return
}
resources = append(resources, string(repo.Key()))
}
} else if published.SourceKind == deb.SourceSnapshot {
for _, uuid := range published.Sources {
snapshot, err2 := snapshotCollection.ByUUID(uuid)
if err2 != nil {
AbortWithJSONError(c, http.StatusNotFound, err2)
return
}
resources = append(resources, string(snapshot.ResourceKey()))
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err))
return
}
if b.SkipContents != nil {
published.SkipContents = *b.SkipContents
}
if b.SkipBz2 != nil {
published.SkipBz2 = *b.SkipBz2
}
if b.AcquireByHash != nil {
published.AcquireByHash = *b.AcquireByHash
}
if b.MultiDist != nil {
published.MultiDist = *b.MultiDist
}
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.PublishedRepoCollection()
published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution)
result, err := published.Update(collectionFactory, out)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = taskCollection.LoadComplete(published, taskCollectionFactory)
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
// Apply field mutations on the freshly loaded object.
if b.SkipContents != nil {
published.SkipContents = *b.SkipContents
}
if b.SkipBz2 != nil {
published.SkipBz2 = *b.SkipBz2
}
if b.AcquireByHash != nil {
published.AcquireByHash = *b.AcquireByHash
}
if b.SignedBy != nil {
published.SignedBy = *b.SignedBy
}
if b.MultiDist != nil {
published.MultiDist = *b.MultiDist
}
if b.Label != nil {
published.Label = *b.Label
}
if b.Origin != nil {
published.Origin = *b.Origin
}
if b.Version != nil {
published.Version = *b.Version
}
result, err := published.Update(taskCollectionFactory, out)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
err = taskCollection.Update(published)
err = collection.Update(published)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err)
}
@@ -1222,7 +1046,7 @@ func apiPublishUpdate(c *gin.Context) {
if b.SkipCleanup == nil || !*b.SkipCleanup {
cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources))
cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...)
err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out)
err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
+114 -228
View File
@@ -24,19 +24,19 @@ import (
// @Tags Repos
// @Produce html
// @Success 200 {object} string "HTML"
// @Router /repos [get]
// @Router /api/repos [get]
func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
c.Writer.Flush()
_, _ = c.Writer.WriteString("<pre>\n")
c.Writer.WriteString("<pre>\n")
if len(localRepos) == 0 {
_, _ = c.Writer.WriteString("<a href=\"-/\">default</a>\n")
c.Writer.WriteString("<a href=\"-/\">default</a>\n")
}
for publishPrefix := range localRepos {
_, _ = c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
}
_, _ = c.Writer.WriteString("</pre>")
c.Writer.WriteString("</pre>")
c.Writer.Flush()
}
}
@@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H
// @Param pkgPath path string true "Package Path" allowReserved=true
// @Produce json
// @Success 200 ""
// @Router /repos/{storage}/{pkgPath} [get]
// @Router /api/{storage}/{pkgPath} [get]
func reposServeInAPIMode(c *gin.Context) {
pkgpath := c.Param("pkgPath")
@@ -69,26 +69,17 @@ func reposServeInAPIMode(c *gin.Context) {
// @Description Each repo is returned as in “show” API.
// @Tags Repos
// @Produce json
// @Success 200 {array} localRepoResponse
// @Success 200 {array} deb.LocalRepo
// @Router /api/repos [get]
func apiReposList(c *gin.Context) {
result := []localRepoResponse{}
result := []*deb.LocalRepo{}
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
err := collection.ForEach(func(r *deb.LocalRepo) error {
err := collection.LoadComplete(r)
if err != nil {
return err
}
result = append(result, newLocalRepoResponse(r))
collection.ForEach(func(r *deb.LocalRepo) error {
result = append(result, r)
return nil
})
if err != nil {
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, result)
}
@@ -102,7 +93,7 @@ type repoCreateParams struct {
DefaultDistribution string ` json:"DefaultDistribution" example:"stable"`
// Default component when publishing from this local repo
DefaultComponent string ` json:"DefaultComponent" example:"main"`
// Snapshot name to create repository from (optional)
// Snapshot name to create repoitory from (optional)
FromSnapshot string ` json:"FromSnapshot" example:""`
}
@@ -116,9 +107,9 @@ type repoCreateParams struct {
// @Description {"Name":"aptly-repo","Comment":"","DefaultDistribution":"","DefaultComponent":""}
// @Description ```
// @Tags Repos
// @Produce json
// @Consume json
// @Param request body repoCreateParams true "Parameters"
// @Produce json
// @Success 201 {object} deb.LocalRepo
// @Failure 404 {object} Error "Source snapshot not found"
// @Failure 409 {object} Error "Local repo already exists"
@@ -131,69 +122,46 @@ func apiReposCreate(c *gin.Context) {
return
}
// Handler: Pre-task validations (shallow)
repo := deb.NewLocalRepo(b.Name, b.Comment)
repo.DefaultComponent = b.DefaultComponent
repo.DefaultDistribution = b.DefaultDistribution
collectionFactory := context.NewCollectionFactory()
if b.FromSnapshot != "" {
var snapshot *deb.Snapshot
snapshotCollection := collectionFactory.SnapshotCollection()
_, err := snapshotCollection.ByName(b.FromSnapshot)
snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err))
return
}
// Just verify it exists - don't load here
}
// Use generated key resource for repo being created
resources := []string{"LocalRepo:" + b.Name}
if b.FromSnapshot != "" {
resources = append(resources, "Snapshot:"+b.FromSnapshot)
}
taskName := fmt.Sprintf("Create repository %s", b.Name)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh collection and check/create ATOMIC inside task
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.LocalRepoCollection()
// Check duplicate inside lock
if _, err := taskCollection.ByName(b.Name); err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil},
fmt.Errorf("local repo with name %s already exists", b.Name)
}
// Create repo
repo := deb.NewLocalRepo(b.Name, b.Comment)
repo.DefaultComponent = b.DefaultComponent
repo.DefaultDistribution = b.DefaultDistribution
if b.FromSnapshot != "" {
snapshotCollection := taskCollectionFactory.SnapshotCollection()
snapshot, err := snapshotCollection.ByName(b.FromSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil},
fmt.Errorf("source snapshot not found: %s", err)
}
err = snapshotCollection.LoadComplete(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil},
fmt.Errorf("unable to load source snapshot: %s", err)
}
repo.UpdateRefList(snapshot.RefList())
}
err := taskCollection.Add(repo)
err = snapshotCollection.LoadComplete(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err))
return
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: repo}, nil
})
repo.UpdateRefList(snapshot.RefList())
}
localRepoCollection := collectionFactory.LocalRepoCollection()
if _, err := localRepoCollection.ByName(b.Name); err == nil {
AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name))
return
}
err := localRepoCollection.Add(repo)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
c.JSON(http.StatusCreated, repo)
}
type reposEditParams struct {
@@ -203,17 +171,15 @@ type reposEditParams struct {
Comment *string ` json:"Comment" example:"example repo"`
// Change Default Distribution for publishing
DefaultDistribution *string ` json:"DefaultDistribution" example:""`
// Change Default Component for publishing
// Change Devault Component for publishing
DefaultComponent *string ` json:"DefaultComponent" example:""`
}
// @Summary Update Repository
// @Description **Update local repository meta information**
// @Tags Repos
// @Param name path string true "Repository name"
// @Consume json
// @Param request body reposEditParams true "Parameters"
// @Produce json
// @Param request body reposEditParams true "Parameters"
// @Success 200 {object} deb.LocalRepo "msg"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Server Error"
@@ -224,67 +190,49 @@ func apiReposEdit(c *gin.Context) {
return
}
// Load shallowly for 404 check and resource key.
// Mutation and duplicate check happen inside the task for atomicity.
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Edit repository %s", name)
if b.Name != nil {
_, err := collection.ByName(*b.Name)
if err == nil {
// already exists
AbortWithJSONError(c, 404, err)
return
}
repo.Name = *b.Name
}
if b.Comment != nil {
repo.Comment = *b.Comment
}
if b.DefaultDistribution != nil {
repo.DefaultDistribution = *b.DefaultDistribution
}
if b.DefaultComponent != nil {
repo.DefaultComponent = *b.DefaultComponent
}
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh collection inside task after lock
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.LocalRepoCollection()
err = collection.Update(repo)
if err != nil {
AbortWithJSONError(c, 500, err)
return
}
// Fresh load after lock acquired
repo, err := taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
}
// Check and update ATOMIC (inside lock)
if b.Name != nil && *b.Name != name {
_, err := taskCollection.ByName(*b.Name)
if err == nil {
// already exists
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil},
fmt.Errorf("local repo with name %q already exists", *b.Name)
}
repo.Name = *b.Name
}
if b.Comment != nil {
repo.Comment = *b.Comment
}
if b.DefaultDistribution != nil {
repo.DefaultDistribution = *b.DefaultDistribution
}
if b.DefaultComponent != nil {
repo.DefaultComponent = *b.DefaultComponent
}
err = taskCollection.Update(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil
})
c.JSON(200, repo)
}
// GET /api/repos/:name
// @Summary Get Repository Info
// @Description Returns basic information about local repository.
// @Tags Repos
// @Param name path string true "Repository name"
// @Produce json
// @Param name path string true "Repository name"
// @Success 200 {object} deb.LocalRepo
// @Failure 404 {object} Error "Repository not found"
// @Router /api/repos/{name} [get]
@@ -306,10 +254,9 @@ func apiReposShow(c *gin.Context) {
// @Description Cannot drop repos that are published.
// @Description Needs force=1 to drop repos used as source by other repos.
// @Tags Repos
// @Param name path string true "Repository name"
// @Produce json
// @Param _async query bool false "Run in background and return task object"
// @Param force query int false "force: 1 to enable"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "Repo object"
// @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Repo Conflict"
@@ -318,10 +265,10 @@ func apiReposDrop(c *gin.Context) {
force := c.Request.URL.Query().Get("force") == "1"
name := c.Params.ByName("name")
// Load shallowly for 404 check, resource key, and task name.
// Full checks (published/snapshots) happen inside the task.
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
snapshotCollection := collectionFactory.SnapshotCollection()
publishedCollection := collectionFactory.PublishedRepoCollection()
repo, err := collection.ByName(name)
if err != nil {
@@ -332,32 +279,19 @@ func apiReposDrop(c *gin.Context) {
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete repo %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh collections inside task after lock acquired
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.LocalRepoCollection()
taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
// Re-read repo with fresh collection after lock
repo, err := taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: %s", err)
}
// Check with fresh collections
published := taskPublishedCollection.ByLocalRepo(repo)
published := publishedCollection.ByLocalRepo(repo)
if len(published) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
}
if !force {
snapshots := taskSnapshotCollection.ByLocalRepoSource(repo)
snapshots := snapshotCollection.ByLocalRepoSource(repo)
if len(snapshots) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override")
}
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, taskCollection.Drop(repo)
return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo)
})
}
@@ -372,12 +306,12 @@ func apiReposDrop(c *gin.Context) {
// @Description ["Pi386 aptly 0.8 966561016b44ed80"]
// @Description ```
// @Tags Repos
// @Param name path string true "Repository name"
// @Produce json
// @Param name path string true "Snapshot to search"
// @Param q query string true "Package query (e.g Name%20(~%20matlab))"
// @Param withDeps query string true "Set to 1 to include dependencies when evaluating package query"
// @Param format query string true "Set to 'details' to return extra info about each package"
// @Param maximumVersion query string true "Set to 1 to only return the highest version for each package name"
// @Produce json
// @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found"
// @Failure 404 {object} Error "Internal Server Error"
@@ -414,13 +348,10 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
return
}
// Load shallowly for 404 check and resource key.
// Full load and mutations happen inside the task.
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
AbortWithJSONError(c, 404, err)
return
@@ -429,23 +360,13 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
resources := []string{string(repo.Key())}
maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh factory and collection inside task after lock
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.LocalRepoCollection()
// Fresh load after lock acquired (use captured `name` variable, not gin context)
repo, err := taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err
}
err = taskCollection.LoadComplete(repo)
err = collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
out.Printf("Loading packages...\n")
list, err := deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -454,7 +375,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
for _, ref := range b.PackageRefs {
var p *deb.Package
p, err = taskCollectionFactory.PackageCollection().ByKey([]byte(ref))
p, err = collectionFactory.PackageCollection().ByKey([]byte(ref))
if err != nil {
if err == database.ErrNotFound {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err)
@@ -470,7 +391,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
err = taskCollection.Update(repo)
err = collectionFactory.LocalRepoCollection().Update(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
}
@@ -485,11 +406,9 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
// @Description
// @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages cant be part of the same local repository.
// @Tags Repos
// @Param name path string true "Repository name"
// @Consume json
// @Produce json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
@@ -507,11 +426,9 @@ func apiReposPackagesAdd(c *gin.Context) {
// @Description
// @Description Any package(s) can be removed from a local repository. Package references from a local repository can be retrieved with GET /api/repos/:name/packages.
// @Tags Repos
// @Param name path string true "Repository name"
// @Param _async query bool false "Run in background and return task object"
// @Consume json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Produce json
// @Param request body reposPackagesAddDeleteParams true "Parameters"
// @Param _async query bool false "Run in background and return task object"
// @Success 200 {object} string "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
@@ -532,7 +449,7 @@ func apiReposPackagesDelete(c *gin.Context) {
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory of packages"
// @Param file path string true "Filename"
// @Param file path string false "Filename (optional)"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {string} string "OK"
@@ -577,8 +494,6 @@ func apiReposPackageFromDir(c *gin.Context) {
return
}
// Load shallowly for 404 check and resource key.
// Full load and mutations happen inside the task.
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
@@ -602,17 +517,7 @@ func apiReposPackageFromDir(c *gin.Context) {
resources := []string{string(repo.Key())}
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh factory and collection inside task after lock
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.LocalRepoCollection()
// Fresh load after lock acquired
repo, err := taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
err = taskCollection.LoadComplete(repo)
err = collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -633,13 +538,13 @@ func apiReposPackageFromDir(c *gin.Context) {
packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter)
list, err = deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil)
list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err)
}
processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(),
taskCollectionFactory.PackageCollection(), reporter, nil, taskCollectionFactory.ChecksumCollection)
collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection)
failedFiles = append(failedFiles, failedFiles2...)
processedFiles = append(processedFiles, otherFiles...)
@@ -649,7 +554,7 @@ func apiReposPackageFromDir(c *gin.Context) {
repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list))
err = taskCollection.Update(repo)
err = collectionFactory.LocalRepoCollection().Update(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
}
@@ -665,7 +570,7 @@ func apiReposPackageFromDir(c *gin.Context) {
}
// atempt to remove dir, if it fails, that's fine: probably it's not empty
_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
os.Remove(filepath.Join(context.UploadPath(), dirParam))
}
if failedFiles == nil {
@@ -702,11 +607,11 @@ type reposCopyPackageParams struct {
// @Summary Copy Package
// @Description Copies a package from a source to destination repository
// @Tags Repos
// @Param name path string true "Destination repo"
// @Param src path string true "Source repo"
// @Produce json
// @Param name path string true "Source repo"
// @Param src path string true "Destination repo"
// @Param file path string true "File/packages to copy"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "msg"
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
@@ -728,8 +633,6 @@ func apiReposCopyPackage(c *gin.Context) {
return
}
// Load shallowly for 404 check and resource keys.
// Full load and mutations happen inside the task.
collectionFactory := context.NewCollectionFactory()
dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
if err != nil {
@@ -753,26 +656,12 @@ func apiReposCopyPackage(c *gin.Context) {
resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh factory and collections inside task after lock
taskCollectionFactory := context.NewCollectionFactory()
// Fresh load of both repos after lock acquired
dstRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(dstRepoName)
err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
}
srcRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(srcRepoName)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
}
err = taskCollectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err)
}
err = taskCollectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err)
}
@@ -785,12 +674,12 @@ func apiReposCopyPackage(c *gin.Context) {
RemovedLines: []string{},
}
dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
}
srcList, err := deb.NewPackageListFromRefList(srcRefList, taskCollectionFactory.PackageCollection(), context.Progress())
srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
}
@@ -858,7 +747,7 @@ func apiReposCopyPackage(c *gin.Context) {
} else {
dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
err = taskCollectionFactory.LocalRepoCollection().Update(dstRepo)
err = collectionFactory.LocalRepoCollection().Update(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
}
@@ -873,15 +762,12 @@ func apiReposCopyPackage(c *gin.Context) {
// @Summary Include File from Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory of packages"
// @Param file path string true "File/packages to include"
// @Produce json
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} string "msg"
// @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir}/{file} [post]
@@ -890,22 +776,26 @@ func apiReposIncludePackageFromFile(c *gin.Context) {
apiReposIncludePackageFromDir(c)
}
type reposIncludePackageFromDirReport struct {
Warnings []string
Added []string
Deleted []string
}
type reposIncludePackageFromDirResponse struct {
Report *aptly.RecordingResultReporter
Report reposIncludePackageFromDirReport
FailedFiles []string
}
// @Summary Include Directory
// @Description Allows automatic processing of .changes file controlling package upload (uploaded using File Upload API) to the local repository. i.e. Exposes repo include command in api.
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory of packages"
// @Produce json
// @Param forceReplace query int false "when value is set to 1, when adding package that conflicts with existing package, remove existing package"
// @Param noRemoveFiles query int false "when value is set to 1, dont remove files that have been imported successfully into repository"
// @Param acceptUnsigned query int false "when value is set to 1, accept unsigned .changes files"
// @Param ignoreSignature query int false "when value is set to 1 disable verification of .changes file signature"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 200 {object} reposIncludePackageFromDirResponse "Response"
// @Failure 404 {object} Error "Not Found"
// @Router /api/repos/{name}/include/{dir} [post]
@@ -946,7 +836,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
}
var resources []string
if len(repoTemplate.Root.Nodes) > 1 {
if len(repoTemplate.Tree.Root.Nodes) > 1 {
resources = append(resources, task.AllLocalReposResourcesKey)
} else {
// repo template string is simple text so only use resource key of specific repository
@@ -961,9 +851,6 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Task: Create fresh factory and collection inside task after lock
taskCollectionFactory := context.NewCollectionFactory()
var (
err error
verifier = context.GetVerifier()
@@ -979,8 +866,8 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter)
_, failedFiles2, err = deb.ImportChangesFiles(
changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier,
repoTemplate, context.Progress(), taskCollectionFactory.LocalRepoCollection(), taskCollectionFactory.PackageCollection(),
context.PackagePool(), taskCollectionFactory.ChecksumCollection, nil, query.Parse)
repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(),
context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse)
failedFiles = append(failedFiles, failedFiles2...)
if err != nil {
@@ -989,7 +876,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
if !noRemoveFiles {
// atempt to remove dir, if it fails, that's fine: probably it's not empty
_ = os.Remove(filepath.Join(context.UploadPath(), dirParam))
os.Remove(filepath.Join(context.UploadPath(), dirParam))
}
if failedFiles == nil {
@@ -1009,10 +896,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", "))
}
ret := reposIncludePackageFromDirResponse{
Report: reporter,
FailedFiles: failedFiles,
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil
return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
"Report": reporter,
"FailedFiles": failedFiles,
}}, nil
})
}
-63
View File
@@ -1,63 +0,0 @@
package api
import (
"bytes"
"encoding/json"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
. "gopkg.in/check.v1"
)
type ReposSuite struct {
APISuite
}
var _ = Suite(&ReposSuite{})
func (s *ReposSuite) TestGetReposIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().LocalRepoCollection()
repo := deb.NewLocalRepo("count-repo-list", "")
repo.UpdateRefList(makePackageRefList(c))
c.Assert(collection.Add(repo), IsNil)
response, err := s.HTTPRequest("GET", "/api/repos", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var repos []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &repos)
c.Assert(err, IsNil)
found := false
for _, repo := range repos {
if repo["Name"] == "count-repo-list" {
found = true
value, ok := repo["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *ReposSuite) TestGetReposReturns500OnCorruptRefList(c *C) {
body, err := json.Marshal(gin.H{"Name": "broken-repo-list"})
c.Assert(err, IsNil)
response, err := s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 201)
collection := s.context.NewCollectionFactory().LocalRepoCollection()
repo, err := collection.ByName("broken-repo-list")
c.Assert(err, IsNil)
putRawDBValue(c, &s.APISuite, repo.RefKey(), []byte("not-msgpack"))
response, err = s.HTTPRequest("GET", "/api/repos", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
}
+6 -10
View File
@@ -2,6 +2,7 @@ package api
import (
"net/http"
"os"
"sync/atomic"
"github.com/aptly-dev/aptly/aptly"
@@ -18,12 +19,6 @@ import (
var context *ctx.AptlyContext
// @Summary Get Metrics
// @Description **Get Prometheus Metrics**
// @Tags Status
// @Produce text/plain
// @Success 200 {string} string Metrics
// @Router /api/metrics [get]
func apiMetricsGet() gin.HandlerFunc {
return func(c *gin.Context) {
countPackagesByRepos()
@@ -61,9 +56,13 @@ func Router(c *ctx.AptlyContext) http.Handler {
router.UseRawPath = true
if c.Config().LogFormat == "json" {
c.StructuredLogging(true)
utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
router.Use(JSONLogger())
} else {
c.StructuredLogging(false)
utils.SetupDefaultLogger(c.Config().LogLevel)
router.Use(gin.Logger())
}
@@ -164,15 +163,12 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/mirrors/:name", apiMirrorsShow)
api.GET("/mirrors/:name/packages", apiMirrorsPackages)
api.POST("/mirrors", apiMirrorsCreate)
api.POST("/mirrors/:name", apiMirrorsEdit)
api.PUT("/mirrors/:name", apiMirrorsUpdate)
api.DELETE("/mirrors/:name", apiMirrorsDrop)
}
{
api.GET("/gpg/keys", apiGPGListKeys)
api.POST("/gpg/key", apiGPGAddKey)
api.DELETE("/gpg/key", apiGPGDeleteKey)
}
{
@@ -224,7 +220,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
api.GET("/graph.:ext", apiGraph)
}
{
api.POST("/db/cleanup", apiDBCleanup)
api.POST("/db/cleanup", apiDbCleanup)
}
{
api.GET("/tasks", apiTasksList)
+44 -124
View File
@@ -20,7 +20,7 @@ import (
// @Description Each snapshot is returned as in “show” API.
// @Tags Snapshots
// @Produce json
// @Success 200 {array} snapshotResponse
// @Success 200 {array} deb.Snapshot
// @Router /api/snapshots [get]
func apiSnapshotsList(c *gin.Context) {
SortMethodString := c.Request.URL.Query().Get("sort")
@@ -32,20 +32,11 @@ func apiSnapshotsList(c *gin.Context) {
SortMethodString = "name"
}
result := []snapshotResponse{}
err := collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
err := collection.LoadComplete(snapshot)
if err != nil {
return err
}
result = append(result, newSnapshotResponse(snapshot))
result := []*deb.Snapshot{}
collection.ForEachSorted(SortMethodString, func(snapshot *deb.Snapshot) error {
result = append(result, snapshot)
return nil
})
if err != nil {
AbortWithJSONError(c, 500, err)
return
}
c.JSON(200, result)
}
@@ -165,7 +156,6 @@ func apiSnapshotsCreate(c *gin.Context) {
}
}
// Phase 1: Pre-task validation (shallow load for 404 checks only)
collectionFactory := context.NewCollectionFactory()
snapshotCollection := collectionFactory.SnapshotCollection()
var resources []string
@@ -183,20 +173,8 @@ func apiSnapshotsCreate(c *gin.Context) {
}
maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh factory
taskCollectionFactory := context.NewCollectionFactory()
taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
taskPackageCollection := taskCollectionFactory.PackageCollection()
// Fresh load of all sources after lock acquired
freshSources := make([]*deb.Snapshot, len(b.SourceSnapshots))
for i := range b.SourceSnapshots {
freshSources[i], err = taskSnapshotCollection.ByName(b.SourceSnapshots[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// LoadComplete on fresh copy
err = taskSnapshotCollection.LoadComplete(freshSources[i])
for i := range sources {
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -204,9 +182,9 @@ func apiSnapshotsCreate(c *gin.Context) {
list := deb.NewPackageList()
// verify package refs and build package list using fresh factory
// verify package refs and build package list
for _, ref := range b.PackageRefs {
p, err := taskPackageCollection.ByKey([]byte(ref))
p, err := collectionFactory.PackageCollection().ByKey([]byte(ref))
if err != nil {
if err == database.ErrNotFound {
return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err)
@@ -219,9 +197,9 @@ func apiSnapshotsCreate(c *gin.Context) {
}
}
snapshot = deb.NewSnapshotFromRefList(b.Name, freshSources, deb.NewPackageRefListFromPackageList(list), b.Description)
snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description)
err = taskSnapshotCollection.Add(snapshot)
err = snapshotCollection.Add(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
}
@@ -239,9 +217,10 @@ type snapshotsCreateFromRepositoryParams struct {
// @Summary Snapshot Repository
// @Description **Create a snapshot of a repository by name**
// @Tags Snapshots
// @Param name path string true "Repository name"
// @Consume json
// @Param request body snapshotsCreateFromRepositoryParams true "Parameters"
// @Param name path string true "Repository name"
// @Param name path string true "Name of the snapshot"
// @Param _async query bool false "Run in background and return task object"
// @Produce json
// @Success 201 {object} deb.Snapshot "Created snapshot object"
@@ -328,7 +307,6 @@ func apiSnapshotsUpdate(c *gin.Context) {
return
}
// Phase 1: Pre-task validation (shallow load for 404 check only)
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.SnapshotCollection()
name := c.Params.ByName("name")
@@ -339,38 +317,14 @@ func apiSnapshotsUpdate(c *gin.Context) {
return
}
// Pre-task validation of new name if provided (skip if renaming to same name)
if b.Name != "" && b.Name != name {
_, err = collection.ByName(b.Name)
if err == nil {
AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name))
return
}
}
resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
taskName := fmt.Sprintf("Update snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh factory
taskCollectionFactory := context.NewCollectionFactory()
taskCollection := taskCollectionFactory.SnapshotCollection()
// Fresh load after lock acquired
snapshot, err = taskCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
_, err := collection.ByName(b.Name)
if err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
}
// Fresh duplicate check inside lock
if b.Name != "" {
_, err := taskCollection.ByName(b.Name)
if err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
}
}
// Update fresh copy
if b.Name != "" {
snapshot.Name = b.Name
}
@@ -379,7 +333,7 @@ func apiSnapshotsUpdate(c *gin.Context) {
snapshot.Description = b.Description
}
err = taskCollection.Update(snapshot)
err = collectionFactory.SnapshotCollection().Update(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -433,9 +387,9 @@ func apiSnapshotsDrop(c *gin.Context) {
name := c.Params.ByName("name")
force := c.Request.URL.Query().Get("force") == "1"
// Phase 1: Pre-task validation (shallow load for 404 check only)
collectionFactory := context.NewCollectionFactory()
snapshotCollection := collectionFactory.SnapshotCollection()
publishedCollection := collectionFactory.PublishedRepoCollection()
snapshot, err := snapshotCollection.ByName(name)
if err != nil {
@@ -445,35 +399,21 @@ func apiSnapshotsDrop(c *gin.Context) {
resources := []string{string(snapshot.ResourceKey())}
taskName := fmt.Sprintf("Delete snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh collections
taskCollectionFactory := context.NewCollectionFactory()
taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection()
// Fresh load after lock acquired
snapshot, err := taskSnapshotCollection.ByName(name)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Fresh checks with current collections
published := taskPublishedCollection.BySnapshot(snapshot)
published := publishedCollection.BySnapshot(snapshot)
if len(published) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published")
}
if !force {
// Using fresh collection for dependency check
snapshots := taskSnapshotCollection.BySnapshotSource(snapshot)
snapshots := snapshotCollection.BySnapshotSource(snapshot)
if len(snapshots) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override")
}
}
err = taskSnapshotCollection.Drop(snapshot)
err = snapshotCollection.Drop(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -615,7 +555,7 @@ func apiSnapshotsMerge(c *gin.Context) {
}
if len(body.Sources) < 1 {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("minimum one source snapshot is required"))
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
return
}
@@ -628,7 +568,6 @@ func apiSnapshotsMerge(c *gin.Context) {
return
}
// Phase 1: Pre-task validation (shallow load for 404 checks only)
collectionFactory := context.NewCollectionFactory()
snapshotCollection := collectionFactory.SnapshotCollection()
@@ -645,43 +584,32 @@ func apiSnapshotsMerge(c *gin.Context) {
}
maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh factory
taskCollectionFactory := context.NewCollectionFactory()
taskSnapshotCollection := taskCollectionFactory.SnapshotCollection()
// Fresh load of all sources inside task
freshSources := make([]*deb.Snapshot, len(body.Sources))
for i := range body.Sources {
freshSources[i], err = taskSnapshotCollection.ByName(body.Sources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// LoadComplete on fresh copy
err = taskSnapshotCollection.LoadComplete(freshSources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
err = snapshotCollection.LoadComplete(sources[0])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Merge using fresh sources
result := freshSources[0].RefList()
for i := 1; i < len(freshSources); i++ {
result = result.Merge(freshSources[i].RefList(), overrideMatching, false)
result := sources[0].RefList()
for i := 1; i < len(sources); i++ {
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
result = result.Merge(sources[i].RefList(), overrideMatching, false)
}
if latest {
result.FilterLatestRefs()
}
sourceDescription := make([]string, len(freshSources))
for i, s := range freshSources {
sourceDescription := make([]string, len(sources))
for i, s := range sources {
sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
}
snapshot = deb.NewSnapshotFromRefList(name, freshSources, result,
snapshot = deb.NewSnapshotFromRefList(name, sources, result,
fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
err = taskCollectionFactory.SnapshotCollection().Add(snapshot)
err = collectionFactory.SnapshotCollection().Add(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
}
@@ -765,29 +693,21 @@ func apiSnapshotsPull(c *gin.Context) {
resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// Phase 2: Inside task lock - create fresh factory
taskCollectionFactory := context.NewCollectionFactory()
// Fresh load of snapshots after lock acquired
freshToSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(name)
err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshSourceSnapshot)
err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// convert snapshots to package list
toPackageList, err := deb.NewPackageListFromRefList(freshToSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
sourcePackageList, err := deb.NewPackageListFromRefList(freshSourceSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress())
sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
@@ -845,7 +765,7 @@ func apiSnapshotsPull(c *gin.Context) {
addedPackages := []string{}
alreadySeen := map[string]bool{}
_ = destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
@@ -861,7 +781,7 @@ func apiSnapshotsPull(c *gin.Context) {
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
_ = toPackageList.Add(pkg)
toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String())
}
@@ -884,10 +804,10 @@ func apiSnapshotsPull(c *gin.Context) {
}
// Create <destination> snapshot
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{freshToSnapshot, freshSourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", freshToSnapshot.Name, freshSourceSnapshot.Name, strings.Join(body.Queries, ", ")))
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
err = taskCollectionFactory.SnapshotCollection().Add(destinationSnapshot)
err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
-53
View File
@@ -1,53 +0,0 @@
package api
import (
"encoding/json"
"github.com/aptly-dev/aptly/deb"
. "gopkg.in/check.v1"
)
type SnapshotsSuite struct {
APISuite
}
var _ = Suite(&SnapshotsSuite{})
func (s *SnapshotsSuite) TestGetSnapshotsIncludesNumPackages(c *C) {
collection := s.context.NewCollectionFactory().SnapshotCollection()
snapshot := deb.NewSnapshotFromRefList("count-snapshot-list", nil, makePackageRefList(c), "")
c.Assert(collection.Add(snapshot), IsNil)
response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 200)
var snapshots []map[string]interface{}
err = json.Unmarshal(response.Body.Bytes(), &snapshots)
c.Assert(err, IsNil)
found := false
for _, snapshot := range snapshots {
if snapshot["Name"] == "count-snapshot-list" {
found = true
value, ok := snapshot["NumPackages"]
c.Assert(ok, Equals, true)
c.Assert(value, Equals, float64(2))
break
}
}
c.Assert(found, Equals, true)
}
func (s *SnapshotsSuite) TestGetSnapshotsReturns500OnCorruptRefList(c *C) {
collection := s.context.NewCollectionFactory().SnapshotCollection()
snapshot := deb.NewSnapshotFromRefList("broken-snapshot-list", nil, makePackageRefList(c), "")
c.Assert(collection.Add(snapshot), IsNil)
putRawDBValue(c, &s.APISuite, snapshot.RefKey(), []byte("not-msgpack"))
response, err := s.HTTPRequest("GET", "/api/snapshots", nil)
c.Assert(err, IsNil)
c.Assert(response.Code, Equals, 500)
c.Assert(response.Body.String(), Matches, ".*msgpack.*|.*decode.*")
}
+1 -1
View File
@@ -1,4 +1,4 @@
package aptly
// AptlyConf holds the default aptly.conf (filled in at link time)
// Default aptly.conf (filled in at link time)
var AptlyConf []byte
+3 -3
View File
@@ -29,7 +29,7 @@ func NewPackagePool(accountName, accountKey, container, prefix, endpoint string)
return &PackagePool{az: azctx}, nil
}
// String returns the storage as string
// String
func (pool *PackagePool) String() string {
return pool.az.String()
}
@@ -104,7 +104,7 @@ func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
if err != nil {
return nil, errors.Wrapf(err, "error creating tempfile for %s", path)
}
defer func() { _ = os.Remove(temp.Name()) }()
defer os.Remove(temp.Name())
_, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil)
if err != nil {
@@ -156,7 +156,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check
if err != nil {
return "", err
}
defer func() { _ = source.Close() }()
defer source.Close()
err = pool.az.putFile(path, source, checksums.MD5)
if err != nil {
+11 -11
View File
@@ -2,12 +2,12 @@ package azure
import (
"context"
"io"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
@@ -50,10 +50,10 @@ func (s *PackagePoolSuite) SetUpTest(c *C) {
s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil)
s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -69,8 +69,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
@@ -81,8 +81,8 @@ func (s *PackagePoolSuite) TestFilepathList(c *C) {
}
func (s *PackagePoolSuite) TestRemove(c *C) {
_, _ = s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
_, _ = s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
c.Check(err, IsNil)
@@ -247,7 +247,7 @@ func (s *PackagePoolSuite) TestOpen(c *C) {
f, err := s.pool.Open(path)
c.Assert(err, IsNil)
contents, err := io.ReadAll(f)
contents, err := ioutil.ReadAll(f)
c.Assert(err, IsNil)
c.Check(len(contents), Equals, 2738)
c.Check(f.Close(), IsNil)
+5 -7
View File
@@ -18,7 +18,7 @@ import (
// PublishedStorage abstract file system with published files (actually hosted on Azure)
type PublishedStorage struct {
// FIXME: unused ???? prefix string
prefix string
az *azContext
pathCache map[string]map[string]string
}
@@ -38,7 +38,7 @@ func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint st
return &PublishedStorage{az: azctx}, nil
}
// String returns the storage as string
// String
func (storage *PublishedStorage) String() string {
return storage.az.String()
}
@@ -65,7 +65,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
if err != nil {
return err
}
defer func() { _ = source.Close() }()
defer source.Close()
err = storage.az.putFile(path, source, sourceMD5)
if err != nil {
@@ -158,7 +158,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath,
if err != nil {
return err
}
defer func() { _ = source.Close() }()
defer source.Close()
err = storage.az.putFile(relFilePath, source, sourceMD5)
if err == nil {
@@ -193,9 +193,7 @@ func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadat
if err != nil {
return fmt.Errorf("error acquiring lease on source blob %s", src)
}
defer func() {
_, _ = blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
}()
defer blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))})
dstBlobClient := containerClient.NewBlobClient(dst)
copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{
+32 -32
View File
@@ -1,17 +1,17 @@
package azure
import (
"bytes"
"context"
"crypto/md5"
"crypto/rand"
"io"
"io/ioutil"
"os"
"path/filepath"
"bytes"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
@@ -36,7 +36,7 @@ func randString(n int) string {
}
const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
var bytes = make([]byte, n)
_, _ = rand.Read(bytes)
rand.Read(bytes)
for i, b := range bytes {
bytes[i] = alphanum[b%byte(len(alphanum))]
}
@@ -69,10 +69,10 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
publicAccessType := azblob.PublicAccessTypeContainer
_, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{
Access: &publicAccessType,
})
c.Assert(err, IsNil)
s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint)
@@ -80,39 +80,39 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
}
func (s *PublishedStorageSuite) TearDownTest(c *C) {
_, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
_, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil)
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil)
c.Assert(err, IsNil)
data, err := io.ReadAll(resp.Body)
data, err := ioutil.ReadAll(resp.Body)
c.Assert(err, IsNil)
return data
}
func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
serviceClient := s.storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(s.storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
serviceClient := s.storage.az.client.ServiceClient()
containerClient := serviceClient.NewContainerClient(s.storage.az.container)
blobClient := containerClient.NewBlobClient(path)
_, err := blobClient.GetProperties(context.Background(), nil)
c.Assert(err, NotNil)
storageError, ok := err.(*azcore.ResponseError)
storageError, ok := err.(*azcore.ResponseError)
c.Assert(ok, Equals, true)
c.Assert(storageError.StatusCode, Equals, 404)
}
func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
hash := md5.Sum(data)
uploadOptions := &azblob.UploadStreamOptions{
HTTPHeaders: &blob.HTTPHeaders{
BlobContentMD5: hash[:],
},
}
reader := bytes.NewReader(data)
_, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
uploadOptions := &azblob.UploadStreamOptions{
HTTPHeaders: &blob.HTTPHeaders{
BlobContentMD5: hash[:],
},
}
reader := bytes.NewReader(data)
_, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions)
c.Assert(err, IsNil)
}
@@ -121,7 +121,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
filename := "a/b.txt"
dir := c.MkDir()
err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
c.Assert(err, IsNil)
err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -140,7 +140,7 @@ func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
filename := "a/b+c.txt"
dir := c.MkDir()
err := os.WriteFile(filepath.Join(dir, "a"), content, 0644)
err := ioutil.WriteFile(filepath.Join(dir, "a"), content, 0644)
c.Assert(err, IsNil)
err = s.storage.PutFile(filename, filepath.Join(dir, "a"))
@@ -258,7 +258,7 @@ func (s *PublishedStorageSuite) TestRemoveDirsPlus(c *C) {
func (s *PublishedStorageSuite) TestRenameFile(c *C) {
dir := c.MkDir()
err := os.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
err := ioutil.WriteFile(filepath.Join(dir, "a"), []byte("Welcome to Azure!"), 0644)
c.Assert(err, IsNil)
err = s.storage.PutFile("source.txt", filepath.Join(dir, "a"))
@@ -280,18 +280,18 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
cs := files.NewMockChecksumStorage()
tmpFile1 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
err := os.WriteFile(tmpFile1, []byte("Contents"), 0644)
err := ioutil.WriteFile(tmpFile1, []byte("Contents"), 0644)
c.Assert(err, IsNil)
cksum1 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
tmpFile2 := filepath.Join(c.MkDir(), "mars-invaders_1.03.deb")
err = os.WriteFile(tmpFile2, []byte("Spam"), 0644)
err = ioutil.WriteFile(tmpFile2, []byte("Spam"), 0644)
c.Assert(err, IsNil)
cksum2 := utils.ChecksumInfo{MD5: "e9dfd31cc505d51fc26975250750deab"}
tmpFile3 := filepath.Join(c.MkDir(), "netboot/boot.img.gz")
_ = os.MkdirAll(filepath.Dir(tmpFile3), 0777)
err = os.WriteFile(tmpFile3, []byte("Contents"), 0644)
os.MkdirAll(filepath.Dir(tmpFile3), 0777)
err = ioutil.WriteFile(tmpFile3, []byte("Contents"), 0644)
c.Assert(err, IsNil)
cksum3 := utils.ChecksumInfo{MD5: "c1df1da7a1ce305a3b60af9d5733ac1d"}
+4 -4
View File
@@ -46,7 +46,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
}
if err == nil && len(listeners) == 1 {
listener := listeners[0]
defer func() { _ = listener.Close() }()
defer listener.Close()
fmt.Printf("\nTaking over web server at: %s (press Ctrl+C to quit)...\n", listener.Addr().String())
err = http.Serve(listener, api.Router(context))
if err != nil {
@@ -67,7 +67,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
if _, ok := <-sigchan; ok {
fmt.Printf("\nShutdown signal received, waiting for background tasks...\n")
context.TaskList().Wait()
_ = server.Shutdown(stdcontext.Background())
server.Shutdown(stdcontext.Background())
}
})()
defer close(sigchan)
@@ -75,14 +75,14 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
listenURL, err := url.Parse(listen)
if err == nil && listenURL.Scheme == "unix" {
file := listenURL.Path
_ = os.Remove(file)
os.Remove(file)
var listener net.Listener
listener, err = net.Listen("unix", file)
if err != nil {
return fmt.Errorf("failed to listen on: %s\n%s", file, err)
}
defer func() { _ = listener.Close() }()
defer listener.Close()
err = server.Serve(listener)
} else {
+1 -1
View File
@@ -97,7 +97,7 @@ package environment to new version.`,
Flag: *flag.NewFlagSet("aptly", flag.ExitOnError),
Subcommands: []*commander.Command{
makeCmdConfig(),
makeCmdDB(),
makeCmdDb(),
makeCmdGraph(),
makeCmdMirror(),
makeCmdRepo(),
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"fmt"
"github.com/smira/commander"
yaml "gopkg.in/yaml.v3"
"gopkg.in/yaml.v3"
)
func aptlyConfigShow(_ *commander.Command, _ []string) error {
+3 -3
View File
@@ -4,13 +4,13 @@ import (
"github.com/smira/commander"
)
func makeCmdDB() *commander.Command {
func makeCmdDb() *commander.Command {
return &commander.Command{
UsageLine: "db",
Short: "manage aptly's internal database and package pool",
Subcommands: []*commander.Command{
makeCmdDBCleanup(),
makeCmdDBRecover(),
makeCmdDbCleanup(),
makeCmdDbRecover(),
},
}
}
+7 -18
View File
@@ -12,7 +12,7 @@ import (
)
// aptly db cleanup
func aptlyDBCleanup(cmd *commander.Command, args []string) error {
func aptlyDbCleanup(cmd *commander.Command, args []string) error {
var err error
if len(args) != 0 {
@@ -26,7 +26,6 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
// collect information about references packages...
existingPackageRefs := deb.NewPackageRefList()
referencedAppStreamFiles := []string{}
// used only in verbose mode to report package use source
packageRefSources := map[string][]string{}
@@ -49,17 +48,13 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose {
description := fmt.Sprintf("mirror %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error {
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
}
for _, poolPath := range repo.AppStreamFiles {
referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
}
return nil
})
if err != nil {
@@ -86,7 +81,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose {
description := fmt.Sprintf("local repo %s", repo.Name)
_ = repo.RefList().ForEach(func(key []byte) error {
repo.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
@@ -118,16 +113,11 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose {
description := fmt.Sprintf("snapshot %s", snapshot.Name)
_ = snapshot.RefList().ForEach(func(key []byte) error {
snapshot.RefList().ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
}
for _, poolPath := range snapshot.AppStreamFiles {
referencedAppStreamFiles = append(referencedAppStreamFiles, poolPath)
}
return nil
})
if err != nil {
@@ -156,7 +146,7 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
if verbose {
description := fmt.Sprintf("published repository %s:%s/%s component %s",
published.Storage, published.Prefix, published.Distribution, component)
_ = published.RefList(component).ForEach(func(key []byte) error {
published.RefList(component).ForEach(func(key []byte) error {
packageRefSources[string(key)] = append(packageRefSources[string(key)], description)
return nil
})
@@ -246,7 +236,6 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
return err
}
referencedFiles = append(referencedFiles, referencedAppStreamFiles...)
sort.Strings(referencedFiles)
context.Progress().ShutdownBar()
@@ -302,9 +291,9 @@ func aptlyDBCleanup(cmd *commander.Command, args []string) error {
return err
}
func makeCmdDBCleanup() *commander.Command {
func makeCmdDbCleanup() *commander.Command {
cmd := &commander.Command{
Run: aptlyDBCleanup,
Run: aptlyDbCleanup,
UsageLine: "cleanup",
Short: "cleanup DB and package pool",
Long: `
+4 -45
View File
@@ -1,16 +1,13 @@
package cmd
import (
"fmt"
"github.com/aptly-dev/aptly/deb"
"github.com/smira/commander"
"github.com/aptly-dev/aptly/database/goleveldb"
)
// aptly db recover
func aptlyDBRecover(cmd *commander.Command, args []string) error {
func aptlyDbRecover(cmd *commander.Command, args []string) error {
var err error
if len(args) != 0 {
@@ -19,19 +16,14 @@ func aptlyDBRecover(cmd *commander.Command, args []string) error {
}
context.Progress().Printf("Recovering database...\n")
if err = goleveldb.RecoverDB(context.DBPath()); err != nil {
return err
}
context.Progress().Printf("Checking database integrity...\n")
err = checkIntegrity()
err = goleveldb.RecoverDB(context.DBPath())
return err
}
func makeCmdDBRecover() *commander.Command {
func makeCmdDbRecover() *commander.Command {
cmd := &commander.Command{
Run: aptlyDBRecover,
Run: aptlyDbRecover,
UsageLine: "recover",
Short: "recover DB after crash",
Long: `
@@ -46,36 +38,3 @@ Example:
return cmd
}
func checkIntegrity() error {
return context.NewCollectionFactory().LocalRepoCollection().ForEach(checkRepo)
}
func checkRepo(repo *deb.LocalRepo) error {
collectionFactory := context.NewCollectionFactory()
repos := collectionFactory.LocalRepoCollection()
err := repos.LoadComplete(repo)
if err != nil {
return fmt.Errorf("load complete repo %q: %s", repo.Name, err)
}
dangling, err := deb.FindDanglingReferences(repo.RefList(), collectionFactory.PackageCollection())
if err != nil {
return fmt.Errorf("find dangling references: %w", err)
}
if len(dangling.Refs) > 0 {
for _, ref := range dangling.Refs {
context.Progress().Printf("Removing dangling database reference %q\n", ref)
}
repo.UpdateRefList(repo.RefList().Subtract(dangling))
if err = repos.Update(repo); err != nil {
return fmt.Errorf("update repo: %w", err)
}
}
return nil
}
+2 -2
View File
@@ -38,8 +38,8 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
if err != nil {
return err
}
_ = tempfile.Close()
_ = os.Remove(tempfile.Name())
tempfile.Close()
os.Remove(tempfile.Name())
format := context.Flags().Lookup("format").Value.String()
output := context.Flags().Lookup("output").Value.String()
+1 -1
View File
@@ -20,7 +20,7 @@ func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
verifier.AddKeyring(keyRing)
}
err := verifier.InitKeyring(!ignoreSignatures) // be verbose only if verifying signatures is requested
err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
if err != nil {
return nil, err
}
+1 -3
View File
@@ -20,7 +20,6 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
downloadAppStream := context.Flags().Lookup("with-appstream").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
@@ -42,7 +41,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
}
repo, err := deb.NewRemoteRepo(mirrorName, archiveURL, distribution, components, context.ArchitecturesList(),
downloadSources, downloadUdebs, downloadInstaller, downloadAppStream)
downloadSources, downloadUdebs, downloadInstaller)
if err != nil {
return fmt.Errorf("unable to create mirror: %s", err)
}
@@ -101,7 +100,6 @@ Example:
}
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
-7
View File
@@ -35,8 +35,6 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
repo.Filter = flag.Value.String() // allows file/stdin with @
case "filter-with-deps":
repo.FilterWithDeps = flag.Value.Get().(bool)
case "with-appstream":
repo.DownloadAppStream = flag.Value.Get().(bool)
case "with-installer":
repo.DownloadInstaller = flag.Value.Get().(bool)
case "with-sources":
@@ -55,10 +53,6 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to edit: flat mirrors don't support udebs")
}
if repo.IsFlat() && repo.DownloadAppStream {
return fmt.Errorf("unable to edit: flat mirrors don't support AppStream (DEP-11) metadata")
}
if repo.Filter != "" {
_, err = query.Parse(repo.Filter)
if err != nil {
@@ -113,7 +107,6 @@ Example:
AddStringOrFileFlag(&cmd.Flag, "filter", "", "filter packages in mirror, use '@file' to read filter from file or '@-' for stdin")
cmd.Flag.Bool("filter-with-deps", false, "when filtering, include dependencies of matching packages as well")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("with-appstream", false, "download AppStream (DEP-11) metadata")
cmd.Flag.Bool("with-installer", false, "download additional not packaged installer files")
cmd.Flag.Bool("with-sources", false, "download source packages in addition to binary packages")
cmd.Flag.Bool("with-udebs", false, "download .udeb packages (Debian installer support)")
+4 -4
View File
@@ -32,7 +32,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
repos := make([]string, collectionFactory.RemoteRepoCollection().Len())
i := 0
_ = collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
collectionFactory.RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
if raw {
repos[i] = repo.Name
} else {
@@ -42,7 +42,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
return nil
})
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Strings(repos)
@@ -70,13 +70,13 @@ func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
i := 0
_ = context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
context.NewCollectionFactory().RemoteRepoCollection().ForEach(func(repo *deb.RemoteRepo) error {
repos[i] = repo
i++
return nil
})
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool {
return repos[i].Name < repos[j].Name
+2 -7
View File
@@ -61,11 +61,6 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
downloadUdebs = Yes
}
fmt.Printf("Download .udebs: %s\n", downloadUdebs)
downloadAppStream := No
if repo.DownloadAppStream {
downloadAppStream = Yes
}
fmt.Printf("Download AppStream: %s\n", downloadAppStream)
if repo.Filter != "" {
fmt.Printf("Filter: %s\n", repo.Filter)
filterWithDeps := No
@@ -91,7 +86,7 @@ func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
if repo.LastDownloadDate.IsZero() {
fmt.Printf("Unable to show package list, mirror hasn't been downloaded yet.\n")
} else {
_ = ListPackagesRefList(repo.RefList(), collectionFactory)
ListPackagesRefList(repo.RefList(), collectionFactory)
}
}
@@ -124,7 +119,7 @@ func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
}
list.PrepareIndex()
_ = list.ForEachIndexed(func(p *deb.Package) error {
list.ForEachIndexed(func(p *deb.Package) error {
repo.Packages = append(repo.Packages, p.GetFullName())
return nil
})
+4 -15
View File
@@ -64,15 +64,6 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err)
}
if repo.DownloadAppStream && !repo.IsFlat() {
context.Progress().Printf("Downloading AppStream metadata...\n")
err = repo.DownloadAppStreamFiles(context.Progress(), context.Downloader(),
context.PackagePool(), collectionFactory.ChecksumCollection(nil), ignoreChecksums)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
}
if repo.Filter != "" {
context.Progress().Printf("Applying filter...\n")
var filterQuery deb.PackageQuery
@@ -96,11 +87,10 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
)
skipExistingPackages := context.Flags().Lookup("skip-existing-packages").Value.Get().(bool)
latestOnly := context.Flags().Lookup("latest").Value.Get().(bool)
context.Progress().Printf("Building download queue...\n")
queue, downloadSize, err = repo.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(),
collectionFactory.ChecksumCollection(nil), skipExistingPackages, latestOnly)
collectionFactory.ChecksumCollection(nil), skipExistingPackages)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
@@ -111,7 +101,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
err = context.ReOpenDatabase()
if err == nil {
repo.MarkAsIdle()
_ = collectionFactory.RemoteRepoCollection().Update(repo)
collectionFactory.RemoteRepoCollection().Update(repo)
}
}()
@@ -183,7 +173,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
_ = file.Close()
file.Close()
}
}
if e != nil {
@@ -271,7 +261,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n "))
}
_ = repo.FinalizeDownload(collectionFactory, context.Progress())
repo.FinalizeDownload(collectionFactory, context.Progress())
err = collectionFactory.RemoteRepoCollection().Update(repo)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
@@ -302,7 +292,6 @@ Example:
cmd.Flag.Bool("ignore-checksums", false, "ignore checksum mismatches while downloading package files and metadata")
cmd.Flag.Bool("ignore-signatures", false, "disable verification of Release file signatures")
cmd.Flag.Bool("skip-existing-packages", false, "do not check file existence for packages listed in the internal database of the mirror")
cmd.Flag.Bool("latest", false, "download only latest version of each package (per architecture)")
cmd.Flag.Int64("download-limit", 0, "limit download speed (kbytes/sec)")
cmd.Flag.String("downloader", "default", "downloader to use (e.g. grab)")
cmd.Flag.Int("max-tries", 1, "max download tries till process fails with download error")
+1 -1
View File
@@ -40,7 +40,7 @@ func aptlyPackageSearch(cmd *commander.Command, args []string) error {
}
format := context.Flags().Lookup("format").Value.String()
_ = PrintPackageList(result, format, "")
PrintPackageList(result, format, "")
return err
}
+3 -3
View File
@@ -84,8 +84,8 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
result := q.Query(collectionFactory.PackageCollection())
err = result.ForEach(func(p *deb.Package) error {
_ = p.Stanza().WriteTo(w, p.IsSource, false, false)
_ = w.Flush()
p.Stanza().WriteTo(w, p.IsSource, false, false)
w.Flush()
fmt.Printf("\n")
if withFiles {
@@ -109,7 +109,7 @@ func aptlyPackageShow(cmd *commander.Command, args []string) error {
if withReferences {
fmt.Printf("References to package:\n")
_ = printReferencesTo(p, collectionFactory)
printReferencesTo(p, collectionFactory)
fmt.Printf("\n")
}
+1 -33
View File
@@ -1,8 +1,6 @@
package cmd
import (
"strings"
"github.com/aptly-dev/aptly/pgp"
"github.com/smira/commander"
"github.com/smira/flag"
@@ -14,20 +12,7 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
}
signer := context.GetSigner()
var gpgKeys []string
// CLI args have priority over config
cliKeys := flags.Lookup("gpg-key").Value.Get().([]string)
if len(cliKeys) > 0 {
gpgKeys = cliKeys
} else if len(context.Config().GpgKeys) > 0 {
gpgKeys = context.Config().GpgKeys
}
for _, gpgKey := range gpgKeys {
signer.SetKey(gpgKey)
}
signer.SetKey(flags.Lookup("gpg-key").Value.String())
signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String())
signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String())
signer.SetBatch(flags.Lookup("batch").Value.Get().(bool))
@@ -41,23 +26,6 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) {
}
type gpgKeyFlag struct {
gpgKeys []string
}
func (k *gpgKeyFlag) Set(value string) error {
k.gpgKeys = append(k.gpgKeys, value)
return nil
}
func (k *gpgKeyFlag) Get() interface{} {
return k.gpgKeys
}
func (k *gpgKeyFlag) String() string {
return strings.Join(k.gpgKeys, ",")
}
func makeCmdPublish() *commander.Command {
return &commander.Command{
UsageLine: "publish",
+2 -2
View File
@@ -53,7 +53,7 @@ func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
return fmt.Errorf("unable to load list of repos: %s", err)
}
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Strings(published)
@@ -99,7 +99,7 @@ func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
return fmt.Errorf("unable to load list of repos: %s", err)
}
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool {
return repos[i].GetPath() < repos[j].GetPath()
+1 -3
View File
@@ -34,7 +34,7 @@ Example:
}
cmd.Flag.String("distribution", "", "distribution name to publish")
cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -51,9 +51,7 @@ Example:
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
cmd.Flag.String("version", "", "version of the release")
return cmd
}
+2 -12
View File
@@ -150,21 +150,13 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
published.AcquireByHash = context.Flags().Lookup("acquire-by-hash").Value.Get().(bool)
}
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
duplicate := collectionFactory.PublishedRepoCollection().CheckDuplicate(published)
if duplicate != nil {
_ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory)
return fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate)
}
@@ -238,7 +230,7 @@ Example:
}
cmd.Flag.String("distribution", "", "distribution name to publish")
cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)")
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -255,8 +247,6 @@ Example:
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.String("version", "", "version of the release")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
+1 -11
View File
@@ -99,14 +99,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
@@ -159,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh
`,
Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError),
}
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -170,8 +162,6 @@ This command would switch published repository (with one component) named ppa/wh
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.String("version", "", "version of the release")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
+1 -21
View File
@@ -60,26 +60,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
if context.Flags().IsSet("signed-by") {
published.SignedBy = context.Flags().Lookup("signed-by").Value.String()
}
if context.Flags().IsSet("origin") {
published.Origin = context.Flags().Lookup("origin").Value.String()
}
if context.Flags().IsSet("label") {
published.Label = context.Flags().Lookup("label").Value.String()
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
if context.Flags().IsSet("version") {
published.Version = context.Flags().Lookup("version").Value.String()
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite, context.SkelPath())
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
@@ -131,7 +115,7 @@ Example:
`,
Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError),
}
cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)")
cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release")
cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)")
cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)")
cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)")
@@ -141,12 +125,8 @@ Example:
cmd.Flag.Bool("skip-contents", false, "don't generate Contents indexes")
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.String("signed-by", "", "an optional field containing a comma separated list of OpenPGP key fingerprints to be used for validating the next Release file")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
cmd.Flag.String("origin", "", "overwrite origin name to publish")
cmd.Flag.String("label", "", "overwrite label to publish")
cmd.Flag.String("version", "", "version of the release")
return cmd
}
+4 -4
View File
@@ -32,7 +32,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
collectionFactory := context.NewCollectionFactory()
repos := make([]string, collectionFactory.LocalRepoCollection().Len())
i := 0
_ = collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
collectionFactory.LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
if raw {
repos[i] = repo.Name
} else {
@@ -47,7 +47,7 @@ func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
return nil
})
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Strings(repos)
@@ -76,7 +76,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
i := 0
_ = context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
context.NewCollectionFactory().LocalRepoCollection().ForEach(func(repo *deb.LocalRepo) error {
e := context.NewCollectionFactory().LocalRepoCollection().LoadComplete(repo)
if e != nil {
return e
@@ -87,7 +87,7 @@ func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
return nil
})
_ = context.CloseDatabase()
context.CloseDatabase()
sort.Slice(repos, func(i, j int) bool {
return repos[i].Name < repos[j].Name
+1 -1
View File
@@ -54,7 +54,7 @@ func aptlyRepoRemove(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to remove: %s", err)
}
_ = toRemove.ForEach(func(p *deb.Package) error {
toRemove.ForEach(func(p *deb.Package) error {
list.Remove(p)
context.Progress().ColoredPrintf("@r[-]@| %s removed", p)
return nil
+1 -1
View File
@@ -52,7 +52,7 @@ func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
if withPackages {
_ = ListPackagesRefList(repo.RefList(), collectionFactory)
ListPackagesRefList(repo.RefList(), collectionFactory)
}
return err
+2 -2
View File
@@ -33,7 +33,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
collection := collectionFactory.SnapshotCollection()
if raw {
_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
fmt.Printf("%s\n", snapshot.Name)
return nil
})
@@ -68,7 +68,7 @@ func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
jsonSnapshots := make([]*deb.Snapshot, collection.Len())
i := 0
_ = collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
collection.ForEachSorted(sortMethodString, func(snapshot *deb.Snapshot) error {
jsonSnapshots[i] = snapshot
i++
return nil
+2 -2
View File
@@ -116,7 +116,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
alreadySeen := map[string]bool{}
_ = result.ForEachIndexed(func(pkg *deb.Package) error {
result.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
@@ -132,7 +132,7 @@ func aptlySnapshotPull(cmd *commander.Command, args []string) error {
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
_ = packageList.Add(pkg)
packageList.Add(pkg)
context.Progress().ColoredPrintf("@g[+]@| %s added", pkg)
}
+1 -1
View File
@@ -123,7 +123,7 @@ func aptlySnapshotMirrorRepoSearch(cmd *commander.Command, args []string) error
}
format := context.Flags().Lookup("format").Value.String()
_ = PrintPackageList(result, format, "")
PrintPackageList(result, format, "")
return err
}
+2 -2
View File
@@ -79,7 +79,7 @@ func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
withPackages := context.Flags().Lookup("with-packages").Value.Get().(bool)
if withPackages {
_ = ListPackagesRefList(snapshot.RefList(), collectionFactory)
ListPackagesRefList(snapshot.RefList(), collectionFactory)
}
return err
@@ -139,7 +139,7 @@ func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
}
list.PrepareIndex()
_ = list.ForEachIndexed(func(p *deb.Package) error {
list.ForEachIndexed(func(p *deb.Package) error {
snapshot.Packages = append(snapshot.Packages, p.GetFullName())
return nil
})
+2 -2
View File
@@ -6,7 +6,7 @@ import (
"os"
"strings"
shellwords "github.com/mattn/go-shellwords"
"github.com/mattn/go-shellwords"
"github.com/smira/commander"
)
@@ -31,7 +31,7 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
if err != nil {
return err
}
defer func() { _ = file.Close() }()
defer file.Close()
scanner := bufio.NewScanner(file)
-3
View File
@@ -185,7 +185,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
$keyring \
"-with-sources=[download source packages in addition to binary packages]:$bool" \
"-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
"-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
"(-)2:new mirror name: " ":archive url:_urls" ":distribution:($dists)" "*:components:_values -s ' ' components $components"
;;
list)
@@ -212,7 +211,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
$keyring \
"-max-tries=[max download tries till process fails with download error]:number: " \
"-skip-existing-packages=[do not check file existence for packages listed in the internal database of the mirror]:$bool" \
"-latest=[download only latest version of each package (per architecture)]:$bool" \
"(-)2:mirror name:$mirrors"
;;
rename)
@@ -225,7 +223,6 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
"-filter-with-deps=[when filtering, include dependencies of matching packages as well]:$bool" \
"-with-sources=[download source packages in addition to binary packages]:$bool" \
"-with-udebs=[download .udeb packages (Debian installer support)]:$bool" \
"-with-appstream=[download AppStream (DEP-11) metadata]:$bool" \
"(-)2:mirror name:$mirrors"
;;
search)
+12 -48
View File
@@ -22,36 +22,34 @@
__aptly_mirror_list()
{
aptly ${aptly_global_opts[@]} mirror list -raw
aptly mirror list -raw
}
__aptly_repo_list()
{
aptly ${aptly_global_opts[@]} repo list -raw
aptly repo list -raw
}
__aptly_snapshot_list()
{
aptly ${aptly_global_opts[@]} snapshot list -raw
aptly snapshot list -raw
}
__aptly_published_distributions()
{
aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 2 | sort | uniq
aptly publish list -raw | cut -d ' ' -f 2 | sort | uniq
}
__aptly_published_prefixes()
{
aptly ${aptly_global_opts[@]} publish list -raw | cut -d ' ' -f 1 | sort | uniq
aptly publish list -raw | cut -d ' ' -f 1 | sort | uniq
}
__aptly_prefixes_for_distribution()
{
aptly ${aptly_global_opts[@]} publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
aptly publish list -raw | awk -v dist="$1" '{ if (dist == $2) print $1 }' | sort | uniq
}
_aptly()
{
cur="${COMP_WORDS[COMP_CWORD]}"
@@ -59,12 +57,7 @@ _aptly()
prevprev="${COMP_WORDS[COMP_CWORD-2]}"
commands="api config db graph mirror package publish repo serve snapshot task version"
options="-architectures -config -db-open-attempts -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider"
options_without_arg="-dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve"
options_with_arg="-architectures -db-open-attempts -gpg-provider"
options_with_path_arg="-config"
options="-architectures= -config= -db-open-attempts= -dep-follow-all-variants -dep-follow-recommends -dep-follow-source -dep-follow-suggests -dep-verbose-resolve -gpg-provider="
db_subcommands="cleanup recover"
mirror_subcommands="create drop edit show list rename search update"
publish_subcommands="drop list repo snapshot switch update source"
@@ -76,41 +69,12 @@ _aptly()
config_subcommands="show"
api_subcommands="serve"
local cmd subcmd numargs numoptions i aptly_global_opts
local cmd subcmd numargs numoptions i
numargs=0
numoptions=0
for opt in "${options_with_path_arg[@]}"; do
[[ "$prev" == "$opt" ]] || continue
compopt -o filenames 2>/dev/null
_filedir
return 0
done
for (( i=1; i < $COMP_CWORD; i++ )); do
word=${COMP_WORDS[i]}
if [[ "$word" == -*=* ]]; then
for o in "${options[@]}"; do
[[ ${word%%=*} == "$o" ]] && aptly_global_opts+=("$word")
done
else
for o in "${options_with_arg[@]}" ""${options_with_path_arg[@]}"" ; do
if [[ "$word" == "$o" ]]; then
if (( i + 1 < COMP_CWORD )); then
aptly_global_opts+=("$word" "${COMP_WORDS[i+1]}")
else
aptly_global_opts+=("$word")
fi
(( i++ ))
continue 2
fi
done
fi
for o in ${options_without_arg[@]}; do
[[ "$word" == "$o" ]] && aptly_global_opts+=("$word")
done
if [[ -n "$cmd" ]]; then
if [[ ! -n "$subcmd" ]]; then
subcmd=${COMP_WORDS[i]}
@@ -203,7 +167,7 @@ _aptly()
"create")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
COMPREPLY=($(compgen -W "-filter= -filter-with-deps -force-components -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
return 0
fi
fi
@@ -211,7 +175,7 @@ _aptly()
"edit")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-appstream -with-installer -with-sources -with-udebs" -- ${cur}))
COMPREPLY=($(compgen -W "-archive-url= -filter= -filter-with-deps -ignore-signatures -keyring= -with-installer -with-sources -with-udebs" -- ${cur}))
else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi
@@ -263,7 +227,7 @@ _aptly()
"update")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages -latest" -- ${cur}))
COMPREPLY=($(compgen -W "-force -download-limit= -downloader= -ignore-checksums -ignore-signatures -keyring= -skip-existing-packages" -- ${cur}))
else
COMPREPLY=($(compgen -W "$(__aptly_mirror_list)" -- ${cur}))
fi
@@ -375,7 +339,7 @@ _aptly()
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-accept-unsigned -force-replace -ignore-signatures -keyring= -no-remove-files -repo= -uploaders-file=" -- ${cur}))
else
compopt -o filenames 2>/dev/null
comptopt -o filenames 2>/dev/null
COMPREPLY=($(compgen -f -- ${cur}))
return 0
fi
+2 -3
View File
@@ -6,7 +6,6 @@ import (
"strings"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/cheggaaa/pb"
"github.com/rs/zerolog/log"
"github.com/wsxiaoys/terminal/color"
@@ -79,7 +78,7 @@ func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
if p.bar != nil {
panic("bar already initialized")
}
if utils.RunningOnTerminal() {
if RunningOnTerminal() {
p.bar = pb.New(0)
p.bar.Total = count
p.bar.NotPrint = true
@@ -142,7 +141,7 @@ func (p *Progress) PrintfStdErr(msg string, a ...interface{}) {
// ColoredPrintf does printf in colored way + newline
func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
if utils.RunningOnTerminal() {
if RunningOnTerminal() {
p.queue <- printTask{code: codePrint, message: color.Sprintf(msg, a...) + "\n"}
} else {
// stip color marks
+1 -1
View File
@@ -11,7 +11,7 @@ func Test(t *testing.T) {
TestingT(t)
}
type ProgressSuite struct{}
type ProgressSuite struct {}
var _ = Suite(&ProgressSuite{})
+12
View File
@@ -0,0 +1,12 @@
package console
import (
"syscall"
"golang.org/x/term"
)
// RunningOnTerminal checks whether stdout is terminal
func RunningOnTerminal() bool {
return term.IsTerminal(syscall.Stdout)
}
+32 -21
View File
@@ -5,11 +5,13 @@ import (
gocontext "context"
"fmt"
"math/rand"
"net/url"
"os"
"os/signal"
"path/filepath"
"runtime"
"runtime/pprof"
"strconv"
"strings"
"sync"
"syscall"
@@ -21,6 +23,7 @@ import (
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/etcddb"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/database/ssdb"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/http"
@@ -29,6 +32,7 @@ import (
"github.com/aptly-dev/aptly/swift"
"github.com/aptly-dev/aptly/task"
"github.com/aptly-dev/aptly/utils"
"github.com/seefan/gossdb/v2/conf"
"github.com/smira/commander"
"github.com/smira/flag"
)
@@ -115,7 +119,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
if err != nil {
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
_ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
utils.SaveConfigRaw(homeLocation, aptly.AptlyConf)
err = utils.LoadConfig(homeLocation, &utils.Config)
if err != nil {
Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err))
@@ -123,14 +127,6 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
}
}
if utils.Config.LogFormat == "json" {
context.StructuredLogging(true)
utils.SetupJSONLogger(utils.Config.LogLevel, os.Stdout)
} else {
context.StructuredLogging(false)
utils.SetupDefaultLogger(utils.Config.LogLevel)
}
context.configLoaded = true
}
@@ -241,7 +237,7 @@ func (context *AptlyContext) newDownloader(progress aptly.Progress) aptly.Downlo
// If flag is defined prefer it to global setting
maxTries = maxTriesFlag.Value.Get().(int)
}
var downloader = context.config().Downloader
var downloader string = context.config().Downloader
downloaderFlag := context.flags.Lookup("downloader")
if downloaderFlag != nil {
downloader = downloaderFlag.Value.String()
@@ -303,12 +299,27 @@ func (context *AptlyContext) _database() (database.Storage, error) {
switch context.config().DatabaseBackend.Type {
case "leveldb":
dbPath := filepath.Join(context.config().GetRootDir(), "db")
if len(context.config().DatabaseBackend.DBPath) != 0 {
dbPath = context.config().DatabaseBackend.DBPath
if len(context.config().DatabaseBackend.DbPath) != 0 {
dbPath = context.config().DatabaseBackend.DbPath
}
context.database, err = goleveldb.NewDB(dbPath)
case "etcd":
context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
case "ssdb":
var cfg conf.Config
u, e := url.Parse(context.config().DatabaseBackend.URL)
if e != nil {
return nil, e
}
cfg.Port, e = strconv.Atoi(u.Port())
cfg.Host = strings.Split(u.Host, ":")[0]
if e != nil {
return nil, e
}
password, _ := u.User.Password()
cfg.Password = password
context.database, err = ssdb.NewOpenDB(&cfg)
default:
context.database, err = goleveldb.NewDB(context.dbPath())
}
@@ -452,7 +463,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
} else if strings.HasPrefix(name, "azure:") {
params, ok := context.config().AzurePublishRoots[name[6:]]
if !ok {
Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:]))
Fatal(fmt.Errorf("Published Azure storage %v not configured", name[6:]))
}
var err error
@@ -597,17 +608,17 @@ func (context *AptlyContext) Shutdown() {
if aptly.EnableDebug {
if context.fileMemProfile != nil {
_ = pprof.WriteHeapProfile(context.fileMemProfile)
_ = context.fileMemProfile.Close()
pprof.WriteHeapProfile(context.fileMemProfile)
context.fileMemProfile.Close()
context.fileMemProfile = nil
}
if context.fileCPUProfile != nil {
pprof.StopCPUProfile()
_ = context.fileCPUProfile.Close()
context.fileCPUProfile.Close()
context.fileCPUProfile = nil
}
if context.fileMemProfile != nil {
_ = context.fileMemProfile.Close()
context.fileMemProfile.Close()
context.fileMemProfile = nil
}
}
@@ -615,7 +626,7 @@ func (context *AptlyContext) Shutdown() {
context.taskList.Stop()
}
if context.database != nil {
_ = context.database.Close()
context.database.Close()
context.database = nil
}
if context.downloader != nil {
@@ -660,7 +671,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
if err != nil {
return nil, err
}
_ = pprof.StartCPUProfile(context.fileCPUProfile)
pprof.StartCPUProfile(context.fileCPUProfile)
}
memprofile := flags.Lookup("memprofile").Value.String()
@@ -680,7 +691,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
return nil, err
}
_, _ = context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
context.fileMemStats.WriteString("# Time\tHeapSys\tHeapAlloc\tHeapIdle\tHeapReleased\n")
go func() {
var stats runtime.MemStats
@@ -690,7 +701,7 @@ func NewContext(flags *flag.FlagSet) (*AptlyContext, error) {
for {
runtime.ReadMemStats(&stats)
if context.fileMemStats != nil {
_, _ = context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
context.fileMemStats.WriteString(fmt.Sprintf("%d\t%d\t%d\t%d\t%d\n",
(time.Now().UnixNano()-start)/1000000, stats.HeapSys, stats.HeapAlloc, stats.HeapIdle, stats.HeapReleased))
time.Sleep(interval)
} else {
+11 -11
View File
@@ -14,7 +14,8 @@ func Test(t *testing.T) {
}
type EtcDDBSuite struct {
db database.Storage
url string
db database.Storage
}
var _ = Suite(&EtcDDBSuite{})
@@ -66,17 +67,17 @@ func (s *EtcDDBSuite) TestDelete(c *C) {
func (s *EtcDDBSuite) TestByPrefix(c *C) {
//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
@@ -108,7 +109,7 @@ func (s *EtcDDBSuite) TestHasPrefix(c *C) {
//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -123,17 +124,15 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
value2 = []byte("value2")
)
transaction, err := s.db.OpenTransaction()
c.Assert(err, IsNil)
err = s.db.Put(key, value)
c.Assert(err, IsNil)
c.Assert(err, IsNil)
_ = transaction.Put(key2, value2)
transaction.Put(key2, value2)
v, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Check(v, DeepEquals, value)
err = transaction.Delete(key)
err = transaction.Delete(key)
c.Assert(err, IsNil)
_, err = transaction.Get(key2)
@@ -156,3 +155,4 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
_, err = transaction.Get(key)
c.Assert(err, NotNil)
}
+1 -1
View File
@@ -145,7 +145,7 @@ func (s *EtcDStorage) Close() error {
return err
}
// Open returns the database
// Reopen tries to open (re-open) the database
func (s *EtcDStorage) Open() error {
if s.db != nil {
return nil
+2 -1
View File
@@ -67,7 +67,8 @@ func (t *transaction) Commit() (err error) {
// Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() {
t.ops = []clientv3.Op{}
_ = t.tmpdb.Drop()
t.tmpdb.Drop()
return
}
// transaction should implement database.Transaction
+2 -2
View File
@@ -51,8 +51,8 @@ func RecoverDB(path string) error {
return err
}
_ = db.Close()
_ = stor.Close()
db.Close()
stor.Close()
return nil
}
+11 -11
View File
@@ -119,17 +119,17 @@ func (s *LevelDBSuite) TestDelete(c *C) {
func (s *LevelDBSuite) TestByPrefix(c *C) {
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
_ = s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
@@ -161,7 +161,7 @@ func (s *LevelDBSuite) TestHasPrefix(c *C) {
c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
@@ -180,8 +180,8 @@ func (s *LevelDBSuite) TestBatch(c *C) {
c.Assert(err, IsNil)
batch := s.db.CreateBatch()
_ = batch.Put(key2, value2)
_ = batch.Delete(key)
batch.Put(key2, value2)
batch.Delete(key)
v, err := s.db.Get(key)
c.Check(err, IsNil)
@@ -202,9 +202,9 @@ func (s *LevelDBSuite) TestBatch(c *C) {
}
func (s *LevelDBSuite) TestCompactDB(c *C) {
_ = s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
_ = s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
_ = s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.CompactDB(), IsNil)
}
+129
View File
@@ -0,0 +1,129 @@
package ssdb
import (
"fmt"
"github.com/aptly-dev/aptly/database"
"github.com/seefan/gossdb/v2/conf"
"github.com/seefan/gossdb/v2/pool"
)
const (
delOpt = "del"
)
type bWriteData struct {
key []byte
value []byte
opts string
err error
}
type Batch struct {
cfg *conf.Config
// key-value chan
w chan bWriteData
p map[string]interface{}
d []string
db *pool.Client
}
// func internalOpenBatch...
func internalOpenBatch(_ database.Storage) *Batch {
b := &Batch{
w: make(chan bWriteData),
p: make(map[string]interface{}),
}
b.run()
return b
}
func (b *Batch) run() {
go func() {
for {
select {
case w, ok := <-b.w:
{
if !ok {
ssdbLog("ssdb batch write chan closed")
return
}
if w.opts == "write" {
ssdbLog("ssdb batch write")
var err error
if len(b.p) > 0 && len(b.d) == 0 {
err = b.db.MultiSet(b.p)
ssdbLog("ssdb batch set errinfo: ", err)
} else if len(b.d) > 0 && len(b.p) == 0 {
err = b.db.MultiDel(b.d...)
ssdbLog("ssdb batch del errinfo: ", err)
} else if len(b.p) == 0 && len(b.d) == 0 {
err = nil
} else {
err = fmt.Errorf("ssdb batch does not support both put and delete operations")
}
ssdbLog("ssdb batch write errinfo: ", err)
b.w <- bWriteData{
err: err,
}
ssdbLog("ssdb batch write end")
} else {
ssdbLog("ssdb batch", w.opts)
if w.opts == "put" {
b.p[string(w.key)] = w.value
} else if w.opts == delOpt {
b.d = append(b.d, string(w.key))
}
}
}
}
}
}()
}
func (b *Batch) stop() {
ssdbLog("ssdb batch stop")
close(b.w)
}
func (b *Batch) Put(key, value []byte) (err error) {
// err = b.db.Set(string(key), string(value))
w := bWriteData{
key: key,
value: value,
opts: "put",
}
b.w <- w
return nil
}
func (b *Batch) Delete(key []byte) (err error) {
/* err = b.db.Del(string(key))
return */
w := bWriteData{
key: key,
opts: delOpt,
}
b.w <- w
return nil
}
func (b *Batch) Write() (err error) {
defer b.stop()
w := bWriteData{
opts: "write",
}
b.w <- w
result := <-b.w
return result.err
}
// batch should implement database.Batch
var (
_ database.Batch = &Batch{}
)
+62
View File
@@ -0,0 +1,62 @@
package ssdb
import (
"os"
"strconv"
"github.com/aptly-dev/aptly/database"
"github.com/seefan/gossdb/v2"
"github.com/seefan/gossdb/v2/conf"
"github.com/seefan/gossdb/v2/pool"
)
var defaultBufSize = 102400
var defaultPoolSize = 1
func internalOpen(cfg *conf.Config) (*pool.Client, error) {
ssdbLog("internalOpen")
cfg.ReadBufferSize = defaultBufSize
cfg.WriteBufferSize = defaultBufSize
cfg.MaxPoolSize = defaultPoolSize
cfg.PoolSize = defaultPoolSize
cfg.MinPoolSize = defaultPoolSize
cfg.MaxWaitSize = 100 * defaultPoolSize
cfg.RetryEnabled = true
//override by env
if os.Getenv("SSDB_READBUFFERSIZE") != "" {
readBufSize, err := strconv.Atoi(os.Getenv("SSDB_READBUFFERSIZE"))
if err != nil {
cfg.ReadBufferSize = readBufSize
}
}
if os.Getenv("SSDB_WRITEBUFFERSIZE") != "" {
writeBufSize, err := strconv.Atoi(os.Getenv("SSDB_WRITEBUFFERSIZE"))
if err != nil {
cfg.WriteBufferSize = writeBufSize
}
}
var cfgs = []*conf.Config{cfg}
err := gossdb.Start(cfgs...)
if err != nil {
return nil, err
}
return gossdb.NewClient()
}
func NewDB(cfg *conf.Config) (database.Storage, error) {
return &Storage{cfg: cfg}, nil
}
func NewOpenDB(cfg *conf.Config) (database.Storage, error) {
db, err := NewDB(cfg)
if err != nil {
return nil, err
}
return db, db.Open()
}
+274
View File
@@ -0,0 +1,274 @@
package ssdb_test
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"testing"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/ssdb"
"github.com/seefan/gossdb/v2/conf"
. "gopkg.in/check.v1"
)
// Launch gocheck tests
func Test(t *testing.T) {
TestingT(t)
}
func setUpSsdb() error {
setUpStr := `
#!/bin/bash
if [ ! -e /tmp/ssdb-master/ssdb-master ]; then
mkdir -p /tmp/ssdb-master
wget --no-check-certificate https://github.com/ideawu/ssdb/archive/master.zip -O /tmp/ssdb-master/master.zip
cd /tmp/ssdb-master && unzip master && cd ssdb-master && make all
fi
cd /tmp/ssdb-master/ssdb-master && ./ssdb-server -d ssdb.conf -s restart
sleep 2`
tmpShell, err := ioutil.TempFile("/tmp", "ssdbSetup")
if err != nil {
return err
}
defer os.Remove(tmpShell.Name())
_, err = tmpShell.WriteString(setUpStr)
if err != nil {
return err
}
cmd := exec.Command("/bin/bash", tmpShell.Name())
fmt.Println(cmd.String())
output, err := cmd.Output()
fmt.Println(string(output))
if err != nil {
return err
}
return nil
}
func TestMain(m *testing.M) {
setUpSsdb()
m.Run()
}
type SSDBSuite struct {
cfg *conf.Config
db database.Storage
}
var _ = Suite(&SSDBSuite{cfg: &conf.Config{
Host: "127.0.0.1",
Port: 8888,
}})
func (s *SSDBSuite) SetUpTest(c *C) {
var err error
s.db, err = ssdb.NewOpenDB(s.cfg)
c.Assert(err, IsNil)
}
func (s *SSDBSuite) TestSetUpTest(c *C) {
var err error
s.db, err = ssdb.NewOpenDB(s.cfg)
c.Assert(err, IsNil)
}
func (s *SSDBSuite) TestGetPut(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
var err error
err = s.db.Put(key, value)
c.Assert(err, IsNil)
result, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Assert(result, DeepEquals, value)
}
func (s *SSDBSuite) TestTemporaryDelete(c *C) {
fmt.Println("TestTemporaryDelete")
var (
key = []byte("key")
value = []byte("value")
)
temp, err := s.db.CreateTemporary()
c.Assert(err, IsNil)
c.Check(temp.HasPrefix([]byte(nil)), Equals, false)
err = temp.Put(key, value)
c.Assert(err, IsNil)
c.Check(temp.HasPrefix([]byte(nil)), Equals, true)
c.Assert(temp.Close(), IsNil)
c.Assert(temp.Drop(), IsNil)
}
func (s *SSDBSuite) TestDelete(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
err := s.db.Put(key, value)
c.Assert(err, IsNil)
_, err = s.db.Get(key)
c.Assert(err, IsNil)
err = s.db.Delete(key)
c.Assert(err, IsNil)
}
func (s *SSDBSuite) TestByPrefix(c *C) {
//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(len(s.db.FetchByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x01}, {0x02}, {0x03}}))
c.Check(len(s.db.KeysByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}))
s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(len(s.db.FetchByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x01}, {0x02}, {0x03}}))
c.Check(len(s.db.KeysByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}))
s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(len(s.db.FetchByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x01}, {0x02}, {0x03}}))
c.Check(len(s.db.KeysByPrefix([]byte{0x80})), DeepEquals, len([][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}))
keys := [][]byte{}
values := [][]byte{}
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
keys = append(keys, append([]byte(nil), k...))
values = append(values, append([]byte(nil), v...))
return nil
}), IsNil)
c.Check(len(values), DeepEquals, len([][]byte{{0x01}, {0x02}, {0x03}}))
c.Check(len(keys), DeepEquals, len([][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}}))
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
return database.ErrNotFound
}), Equals, database.ErrNotFound)
c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
return database.ErrNotFound
}), IsNil)
c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
}
func (s *SSDBSuite) TestHasPrefix(c *C) {
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
//c.Check(s.db.HasPrefix([]byte("")), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
}
func (s *SSDBSuite) TestTransactionCommit(c *C) {
var (
key = []byte("key")
key2 = []byte("key2")
value = []byte("value")
value2 = []byte("value2")
)
s.db.Delete(key)
s.db.Delete(key2)
transaction, err := s.db.OpenTransaction()
c.Assert(err, IsNil)
defer transaction.Discard()
err = s.db.Put(key, value)
c.Assert(err, IsNil)
v, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Check(v, DeepEquals, value)
err = transaction.Put(key2, value2)
c.Assert(err, IsNil)
v, err = transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value2)
_, err = s.db.Get(key2)
c.Assert(err, ErrorMatches, "key not found")
err = transaction.Delete(key)
c.Assert(err, IsNil)
_, err = transaction.Get(key)
c.Assert(err, ErrorMatches, "key not found")
v, err = s.db.Get(key)
c.Assert(err, IsNil)
c.Check(v, DeepEquals, value)
err = transaction.Commit()
c.Check(err, IsNil)
v, err = s.db.Get(key2)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value2)
_, err = s.db.Get(key)
c.Assert(err, ErrorMatches, "key not found")
}
func (s *SSDBSuite) TestBatch(c *C) {
var (
key = []byte("bkey")
key2 = []byte("bkey2")
value = []byte("bvalue")
value2 = []byte("bvalue2")
)
err := s.db.Put(key, value)
c.Check(err, IsNil)
batch := s.db.CreateBatch()
batch.Put(key2, value2)
v, err := s.db.Get(key)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value)
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
err = batch.Write()
c.Check(err, IsNil)
v, err = s.db.Get(key2)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value2)
batch = s.db.CreateBatch()
batch.Delete(key)
batch.Delete(key2)
c.Check(err, IsNil)
v, err = s.db.Get(key)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value)
c.Check(err, IsNil)
v, err = s.db.Get(key2)
c.Check(err, IsNil)
c.Check(v, DeepEquals, value2)
err = batch.Write()
c.Check(err, IsNil)
_, err = s.db.Get(key2)
c.Check(err, ErrorMatches, "key not found")
_, err = s.db.Get(key)
c.Check(err, ErrorMatches, "key not found")
}
+12
View File
@@ -0,0 +1,12 @@
package ssdb
import (
"fmt"
"os"
)
func ssdbLog(a ...interface{}) {
if os.Getenv("SSDB_DEBUG") != "" {
fmt.Println(a...)
}
}
+183
View File
@@ -0,0 +1,183 @@
package ssdb
import (
"os"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/seefan/gossdb/v2"
"github.com/seefan/gossdb/v2/conf"
"github.com/seefan/gossdb/v2/pool"
)
type Storage struct {
cfg *conf.Config
db *pool.Client
}
// CreateTemporary creates new DB of the same type in temp dir
func (s *Storage) CreateTemporary() (database.Storage, error) {
// use leveldb as temp db
tmpPath := os.Getenv("SSDB_TMPDB_PATH")
if tmpPath == "" {
tmpPath = "/tmp/ssdb_tmpdb_path"
}
gdb, err := goleveldb.NewDB(tmpPath)
if err != nil {
return nil, err
}
return gdb.CreateTemporary()
}
// Get key value from ssdb
func (s *Storage) Get(key []byte) (value []byte, err error) {
// ssdbLog("ssdb origin db get key:", string(key))
getResp, err := s.db.Get(string(key))
if err != nil {
return
}
value = getResp.Bytes()
if len(value) == 0 {
err = database.ErrNotFound
return
}
return
}
// Put saves key to ssdb, if key has the same value in DB already, it is not saved
func (s *Storage) Put(key []byte, value []byte) (err error) {
//ssdbLog("ssdb origin db put key:", string(key), " value: ", string(value))
err = s.db.Set(string(key), value)
if err != nil {
return
}
return
}
// Delete removes key from ssdb
func (s *Storage) Delete(key []byte) (err error) {
//ssdbLog("ssdb origin db del key:", string(key))
err = s.db.Del(string(key))
if err != nil {
return
}
return
}
// KeysByPrefix returns all keys that start with prefix
func (s *Storage) KeysByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0)
getResp, err := s.db.Keys(string(prefix), string(prefix)+"}", -1)
if err != nil {
return nil
}
for _, ev := range getResp {
key := []byte(ev)
keyc := make([]byte, len(key))
copy(keyc, key)
result = append(result, key)
}
return result
}
// FetchByPrefix returns all values with keys that start with prefix
func (s *Storage) FetchByPrefix(prefix []byte) [][]byte {
result := make([][]byte, 0)
getResp, err := s.db.Scan(string(prefix), string(prefix)+"}", -1)
if err != nil {
return nil
}
for _, ev := range getResp {
value := ev.Bytes()
valuec := make([]byte, len(value))
copy(valuec, value)
result = append(result, valuec)
}
return result
}
// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
func (s *Storage) HasPrefix(prefix []byte) bool {
//ssdbLog("HasPrefix", string(prefix), string(prefix)+"}")
getResp, err := s.db.Keys(string(prefix), string(prefix)+"}", -1)
if err != nil {
return false
}
//ssdbLog("HasPrefix", len(getResp))
if len(getResp) > 0 {
return true
}
return false
}
// ProcessByPrefix iterates through all entries where key starts with prefix and calls
// StorageProcessor on key value pair
func (s *Storage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
getResp, err := s.db.Scan(string(prefix), string(prefix)+"}", -1)
if err != nil {
return err
}
for k, v := range getResp {
err := proc([]byte(k), v.Bytes())
if err != nil {
return err
}
}
return nil
}
// Close finishes ssdb connect
func (s *Storage) Close() error {
ssdbLog("ssdb close")
if s.db != nil {
s.db.Close()
s.db = nil
}
gossdb.Shutdown()
return nil
}
// Reopen tries to open (re-open) the database
func (s *Storage) Open() error {
ssdbLog("ssdb open")
if s.db != nil && s.db.IsOpen() {
ssdbLog("ssdb opened")
return nil
}
var err error
s.db, err = internalOpen(s.cfg)
return err
}
// CreateBatch creates a Batch object
func (s *Storage) CreateBatch() database.Batch {
Batch := internalOpenBatch(s)
Batch.cfg = s.cfg
Batch.db = s.db
return Batch
}
// OpenTransaction creates new transaction.
func (s *Storage) OpenTransaction() (database.Transaction, error) {
return internalOpenTransaction(s)
}
// CompactDB compacts database by merging layers
func (s *Storage) CompactDB() error {
return nil
}
// Drop removes all the ssdb files (DANGEROUS!)
func (s *Storage) Drop() error {
return nil
}
// Check interface
var (
_ database.Storage = &Storage{}
)
+188
View File
@@ -0,0 +1,188 @@
package ssdb
import (
"fmt"
"github.com/aptly-dev/aptly/database"
)
type trWriteData struct {
key []byte
value []byte
opts string
err error
}
type trReadData struct {
kv []byte
err error
}
type transaction struct {
// for key-value-operation chan
w chan trWriteData
// key read chan
r chan trReadData
q map[string]trWriteData
t database.Storage
}
// func internalOpenTransaction...
func internalOpenTransaction(t database.Storage) (*transaction, error) {
tr := &transaction{
w: make(chan trWriteData),
r: make(chan trReadData),
q: make(map[string]trWriteData),
t: t,
}
return tr, tr.run()
}
// func run...
func (t *transaction) run() error {
go func() {
for {
select {
case w, ok := <-t.w:
{
if !ok {
ssdbLog("ssdb transaction write chan closed")
return
}
if w.opts == "commit" {
ssdbLog("ssdb transaction commit")
var errs []error
for _, vo := range t.q {
if vo.opts == "put" {
err := t.t.Put(vo.key, vo.value)
if err != nil {
//ssdbLog(err)
errs = append(errs, err)
}
}
if vo.opts == delOpt {
err := t.t.Delete(vo.key)
if err != nil {
errs = append(errs, err)
}
}
}
if len(errs) == 0 {
t.w <- trWriteData{
err: nil,
}
} else {
t.w <- trWriteData{
err: fmt.Errorf("ssdb transaction write errs: %v", errs),
}
}
ssdbLog("ssdb transaction commit end")
} else {
ssdbLog("ssdb transaction", w.opts)
//ssdbLog("ssdb r transaction", w.opts, "key: ", string(w.key), "value: ", string(w.value))
t.q[string(w.key)] = w
}
}
case r, ok := <-t.r:
{
if !ok {
ssdbLog("ssdb transaction read chan closed")
return
}
if rData, ok := t.q[string(r.kv)]; ok {
if rData.opts == delOpt {
// del return not found error
t.r <- trReadData{
kv: nil,
err: database.ErrNotFound,
}
} else {
t.r <- trReadData{
kv: rData.value,
err: nil,
}
}
} else {
v, err := t.t.Get(r.kv)
t.r <- trReadData{
kv: v,
err: err,
}
}
}
}
}
}()
return nil
}
// Get implements database.Reader interface.
func (t *transaction) Get(key []byte) ([]byte, error) {
keyc := make([]byte, len(key))
copy(keyc, key)
r := trReadData{
kv: keyc,
err: nil,
}
t.r <- r
result := <-t.r
return result.kv, result.err
}
// Put implements database.Writer interface.
func (t *transaction) Put(key, value []byte) error {
//ssdbLog("golf*********************ssdb put")
//ssdbLog("ssdb transaction db put key:", string(key), " value: ", string(value))
keyc := make([]byte, len(key))
copy(keyc, key)
valuec := make([]byte, len(value))
copy(valuec, value)
w := trWriteData{
key: keyc,
value: valuec,
opts: "put",
}
t.w <- w
return nil
}
// Delete implements database.Writer interface.
func (t *transaction) Delete(key []byte) error {
//return t.t.Delete(key)
//ssdbLog("golf*********************ssdb del")
keyc := make([]byte, len(key))
copy(keyc, key)
w := trWriteData{
key: keyc,
opts: delOpt,
}
t.w <- w
return nil
}
func (t *transaction) Commit() error {
w := trWriteData{
opts: "commit",
}
t.w <- w
result := <-t.w
return result.err
}
// Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() {
ssdbLog("ssdb transaction stop")
close(t.r)
close(t.w)
}
// transaction should implement database.Transaction
var _ database.Transaction = &transaction{}
+8 -8
View File
@@ -60,14 +60,14 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil {
return err
}
defer func() { _ = input.Close() }()
defer input.Close()
isClearSigned, err := verifier.IsClearSigned(input)
if err != nil {
return err
}
_, _ = input.Seek(0, 0)
input.Seek(0, 0)
if !isClearSigned && !acceptUnsigned {
return fmt.Errorf(".changes file is not signed and unsigned processing hasn't been enabled")
@@ -79,7 +79,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil {
return err
}
_, _ = input.Seek(0, 0)
input.Seek(0, 0)
c.SignatureKeys = keyInfo.GoodKeys
}
@@ -91,7 +91,7 @@ func (c *Changes) VerifyAndParse(acceptUnsigned, ignoreSignature bool, verifier
if err != nil {
return err
}
defer func() { _ = text.Close() }()
defer text.Close()
} else {
text = input
}
@@ -307,7 +307,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup()
changes.Cleanup()
continue
}
@@ -315,7 +315,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup()
changes.Cleanup()
continue
}
@@ -334,7 +334,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
if err != nil {
failedFiles = append(failedFiles, path)
reporter.Warning("unable to process file %s: %s", changes.ChangesName, err)
_ = changes.Cleanup()
changes.Cleanup()
continue
}
@@ -354,7 +354,7 @@ func ImportChangesFiles(changesFiles []string, reporter aptly.ResultReporter, ac
failedFiles = append(failedFiles, path)
reporter.Warning("changes file skipped due to uploaders config: %s, keys %#v: %s",
changes.ChangesName, changes.SignatureKeys, err)
_ = changes.Cleanup()
changes.Cleanup()
continue
}
}
+3 -3
View File
@@ -51,7 +51,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
func (s *ChangesSuite) TearDownTest(c *C) {
s.progress.Shutdown()
_ = s.db.Close()
s.db.Close()
}
func (s *ChangesSuite) TestParseAndVerify(c *C) {
@@ -108,13 +108,13 @@ func (s *ChangesSuite) TestImportChangesFiles(c *C) {
for _, path := range origFailedFiles {
filename := filepath.Join(s.Dir, filepath.Base(path))
_ = utils.CopyFile(path, filename)
utils.CopyFile(path, filename)
expectedFailedFiles = append(expectedFailedFiles, filename)
}
for _, path := range origProcessedFiles {
filename := filepath.Join(s.Dir, filepath.Base(path))
_ = utils.CopyFile(path, filename)
utils.CopyFile(path, filename)
expectedProcessedFiles = append(expectedProcessedFiles, filename)
}
+1 -1
View File
@@ -28,7 +28,7 @@ func (s *ChecksumCollectionSuite) SetUpTest(c *C) {
}
func (s *ChecksumCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close()
s.db.Close()
}
func (s *ChecksumCollectionSuite) TestFlow(c *C) {
+10 -10
View File
@@ -17,7 +17,7 @@ import (
"github.com/aptly-dev/aptly/pgp"
"github.com/kjk/lzma"
"github.com/klauspost/compress/zstd"
xz "github.com/smira/go-xz"
"github.com/smira/go-xz"
)
// Source kinds
@@ -35,7 +35,7 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
defer file.Close()
library := ar.NewReader(file)
for {
@@ -66,14 +66,14 @@ func GetControlFileFromDeb(packageFile string) (Stanza, error) {
if err != nil {
return nil, errors.Wrapf(err, "unable to ungzip %s from %s", header.Name, packageFile)
}
defer func() { _ = ungzip.Close() }()
defer ungzip.Close()
tarInput = ungzip
case "control.tar.xz":
unxz, err := xz.NewReader(bufReader)
if err != nil {
return nil, errors.Wrapf(err, "unable to unxz %s from %s", header.Name, packageFile)
}
defer func() { _ = unxz.Close() }()
defer unxz.Close()
tarInput = unxz
case "control.tar.zst":
unzstd, err := zstd.NewReader(bufReader)
@@ -116,10 +116,10 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
if err != nil {
return nil, err
}
defer func() { _ = file.Close() }()
defer file.Close()
isClearSigned, err := verifier.IsClearSigned(file)
_, _ = file.Seek(0, 0)
file.Seek(0, 0)
if err != nil {
return nil, err
@@ -132,7 +132,7 @@ func GetControlFileFromDsc(dscFile string, verifier pgp.Verifier) (Stanza, error
if err != nil {
return nil, err
}
defer func() { _ = text.Close() }()
defer text.Close()
} else {
text = file
}
@@ -181,7 +181,7 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
if err != nil {
return nil, errors.Wrapf(err, "unable to ungzip data.tar.gz from %s", packageFile)
}
defer func() { _ = ungzip.Close() }()
defer ungzip.Close()
tarInput = ungzip
}
case "data.tar.bz2":
@@ -191,11 +191,11 @@ func GetContentsFromDeb(file io.Reader, packageFile string) ([]string, error) {
if err != nil {
return nil, errors.Wrapf(err, "unable to unxz data.tar.xz from %s", packageFile)
}
defer func() { _ = unxz.Close() }()
defer unxz.Close()
tarInput = unxz
case "data.tar.lzma":
unlzma := lzma.NewReader(bufReader)
defer func() { _ = unlzma.Close() }()
defer unlzma.Close()
tarInput = unlzma
case "data.tar.zst":
unzstd, err := zstd.NewReader(bufReader)
-45
View File
@@ -1,45 +0,0 @@
package deb
import (
"errors"
"fmt"
"github.com/aptly-dev/aptly/database"
)
// FindDanglingReferences finds references that exist in the given PackageRefList, but not in the given PackageCollection.
// It returns all such references, so they can be removed from the database.
func FindDanglingReferences(reflist *PackageRefList, packages *PackageCollection) (dangling *PackageRefList, err error) {
dangling = &PackageRefList{}
err = reflist.ForEach(func(key []byte) error {
ok, err := isDangling(packages, key)
if err != nil {
return err
}
if ok {
dangling.Refs = append(dangling.Refs, key)
}
return nil
})
if err != nil {
return nil, err
}
return dangling, nil
}
func isDangling(packages *PackageCollection, key []byte) (bool, error) {
_, err := packages.ByKey(key)
if errors.Is(err, database.ErrNotFound) {
return true, nil
}
if err != nil {
return false, fmt.Errorf("get reference %q: %w", key, err)
}
return false, nil
}
-46
View File
@@ -1,46 +0,0 @@
package deb_test
import (
"bytes"
"testing"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/deb"
)
func TestFindDanglingReferences(t *testing.T) {
reflist := deb.NewPackageRefList()
reflist.Refs = [][]byte{[]byte("P existing 1.2.3"), []byte("P dangling 1.2.3")}
db, _ := goleveldb.NewOpenDB(t.TempDir())
packages := deb.NewPackageCollection(db)
if err := packages.Update(&deb.Package{Name: "existing", Version: "1.2.3"}); err != nil {
t.Fatal(err)
}
dangling, err := deb.FindDanglingReferences(reflist, packages)
if err != nil {
t.Fatal(err)
}
exp := &deb.PackageRefList{
Refs: [][]byte{[]byte("P dangling 1.2.3")},
}
compareRefs(t, exp, dangling)
}
func compareRefs(t *testing.T, exp, got *deb.PackageRefList) {
t.Helper()
if len(exp.Refs) != len(got.Refs) {
t.Fatalf("refs length mismatch: exp %d, got %d", len(exp.Refs), len(got.Refs))
}
for i := range exp.Refs {
if !bytes.Equal(exp.Refs[i], got.Refs[i]) {
t.Fatalf("refs do not match: exp %q, got %q", exp.Refs[i], got.Refs[i])
}
}
}
-3
View File
@@ -26,11 +26,8 @@ var (
"Version",
"Codename",
"Date",
"Valid-Until",
"NotAutomatic",
"ButAutomaticUpgrades",
"Acquire-By-Hash",
"Signed-By",
"Architectures",
"Architecture",
"Components",
+1 -1
View File
@@ -163,7 +163,7 @@ func (s *ControlFileSuite) TestCanonicalCase(c *C) {
func (s *ControlFileSuite) TestLongFields(c *C) {
f, err := os.Open("long.stanza")
c.Assert(err, IsNil)
defer func() { _ = f.Close() }()
defer f.Close()
r := NewControlFileReader(f, false, false)
stanza, e := r.ReadStanza()
+11 -11
View File
@@ -12,15 +12,15 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
var err error
graph := gographviz.NewEscape()
_ = graph.SetDir(true)
_ = graph.SetName("aptly")
graph.SetDir(true)
graph.SetName("aptly")
var labelStart string
var labelEnd string
switch layout {
case "vertical":
_ = graph.AddAttr("aptly", "rankdir", "LR")
graph.AddAttr("aptly", "rankdir", "LR")
labelStart = ""
labelEnd = ""
case "horizontal":
@@ -38,7 +38,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return e
}
_ = graph.AddNode("aptly", repo.UUID, map[string]string{
graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord",
"style": "filled",
"fillcolor": "darkgoldenrod1",
@@ -60,7 +60,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return e
}
_ = graph.AddNode("aptly", repo.UUID, map[string]string{
graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord",
"style": "filled",
"fillcolor": "mediumseagreen",
@@ -75,7 +75,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return nil, err
}
_ = collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
collectionFactory.SnapshotCollection().ForEach(func(snapshot *Snapshot) error {
existingNodes[snapshot.UUID] = true
return nil
})
@@ -91,7 +91,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
description = "Snapshot from repo"
}
_ = graph.AddNode("aptly", snapshot.UUID, map[string]string{
graph.AddNode("aptly", snapshot.UUID, map[string]string{
"shape": "Mrecord",
"style": "filled",
"fillcolor": "cadetblue1",
@@ -103,7 +103,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
for _, uuid := range snapshot.SourceIDs {
_, exists := existingNodes[uuid]
if exists {
_ = graph.AddEdge(uuid, snapshot.UUID, true, nil)
graph.AddEdge(uuid, snapshot.UUID, true, nil)
}
}
}
@@ -114,8 +114,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
return nil, err
}
_ = collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
_ = graph.AddNode("aptly", repo.UUID, map[string]string{
collectionFactory.PublishedRepoCollection().ForEach(func(repo *PublishedRepo) error {
graph.AddNode("aptly", repo.UUID, map[string]string{
"shape": "Mrecord",
"style": "filled",
"fillcolor": "darkolivegreen1",
@@ -127,7 +127,7 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
for _, uuid := range repo.Sources {
_, exists := existingNodes[uuid]
if exists {
_ = graph.AddEdge(uuid, repo.UUID, true, nil)
graph.AddEdge(uuid, repo.UUID, true, nil)
}
}
+7 -7
View File
@@ -59,24 +59,24 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
if file.discardable {
return nil
}
_, _ = file.BufWriter()
file.BufWriter()
}
err := file.w.Flush()
if err != nil {
_ = file.tempFile.Close()
file.tempFile.Close()
return fmt.Errorf("unable to write to index file: %s", err)
}
if file.compressable {
err = utils.CompressFile(file.tempFile, file.onlyGzip || file.parent.skipBz2)
if err != nil {
_ = file.tempFile.Close()
file.tempFile.Close()
return fmt.Errorf("unable to compress index file: %s", err)
}
}
_ = file.tempFile.Close()
file.tempFile.Close()
exts := []string{""}
cksumExts := exts
@@ -220,11 +220,11 @@ func packageIndexByHash(file *indexFile, ext string, hash string, sum string) er
// If we managed to resolve the link target: delete it. This is the
// oldest physical index file we no longer need. Once we drop our
// old symlink we'll essentially forget about it existing at all.
_ = file.parent.publishedStorage.Remove(linkTarget)
file.parent.publishedStorage.Remove(linkTarget)
}
_ = file.parent.publishedStorage.Remove(oldIndexPath)
file.parent.publishedStorage.Remove(oldIndexPath)
}
_ = file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
file.parent.publishedStorage.RenameFile(indexPath, oldIndexPath)
}
// create symlink
+2 -35
View File
@@ -172,39 +172,6 @@ func (l *PackageList) ForEach(handler func(*Package) error) error {
return err
}
// FilterLatest creates a copy of the package list containing only the
// latest version for each package name/architecture pair.
func (l *PackageList) FilterLatest() (*PackageList, error) {
if l == nil {
return nil, fmt.Errorf("package list is nil")
}
filtered := make(map[string]*Package, l.Len())
err := l.ForEach(func(p *Package) error {
key := p.Architecture + "|" + p.Name
if existing, found := filtered[key]; !found || CompareVersions(p.Version, existing.Version) > 0 {
filtered[key] = p
}
return nil
})
if err != nil {
return nil, err
}
result := NewPackageListWithDuplicates(l.duplicatesAllowed, len(filtered))
for _, pkg := range filtered {
if err = result.Add(pkg); err != nil {
return nil, err
}
}
return result, nil
}
// ForEachIndexed calls handler for each package in list in indexed order
func (l *PackageList) ForEachIndexed(handler func(*Package) error) error {
if !l.indexed {
@@ -471,7 +438,7 @@ func (l *PackageList) Scan(q PackageQuery) (result *PackageList) {
result = NewPackageListWithDuplicates(l.duplicatesAllowed, 0)
for _, pkg := range l.packages {
if q.Matches(pkg) {
_ = result.Add(pkg)
result.Add(pkg)
}
}
@@ -489,7 +456,7 @@ func (l *PackageList) SearchByKey(arch, name, version string) (result *PackageLi
pkg := l.packages["P"+arch+" "+name+" "+version]
if pkg != nil {
_ = result.Add(pkg)
result.Add(pkg)
}
return
+10 -59
View File
@@ -96,7 +96,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
{Name: "dpkg", Version: "1.7", Architecture: "source", SourceArchitecture: "any", IsSource: true, deps: &PackageDependencies{}},
}
for _, p := range s.packages {
_ = s.il.Add(p)
s.il.Add(p)
}
s.il.PrepareIndex()
@@ -110,7 +110,7 @@ func (s *PackageListSuite) SetUpTest(c *C) {
{Name: "app", Version: "3.0", Architecture: "amd64", deps: &PackageDependencies{PreDepends: []string{"dpkg >= 1.6)"}, Depends: []string{"lib (>> 0.9)", "data (>= 1.0)"}}},
}
for _, p := range s.packages2 {
_ = s.il2.Add(p)
s.il2.Add(p)
}
s.il2.PrepareIndex()
@@ -202,8 +202,8 @@ func (s *PackageListSuite) TestRemoveWhenIndexed(c *C) {
}
func (s *PackageListSuite) TestForeach(c *C) {
_ = s.list.Add(s.p1)
_ = s.list.Add(s.p3)
s.list.Add(s.p1)
s.list.Add(s.p3)
Len := 0
err := s.list.ForEach(func(*Package) error {
@@ -232,21 +232,21 @@ func (s *PackageListSuite) TestIndex(c *C) {
}
func (s *PackageListSuite) TestAppend(c *C) {
_ = s.list.Add(s.p1)
_ = s.list.Add(s.p3)
s.list.Add(s.p1)
s.list.Add(s.p3)
err := s.list.Append(s.il)
c.Check(err, IsNil)
c.Check(s.list.Len(), Equals, 16)
list := NewPackageList()
_ = list.Add(s.p4)
list.Add(s.p4)
err = s.list.Append(list)
c.Check(err, ErrorMatches, "package already exists and is different: .*")
s.list.PrepareIndex()
c.Check(func() { _ = s.list.Append(s.il) }, Panics, "Append not supported when indexed")
c.Check(func() { s.list.Append(s.il) }, Panics, "Append not supported when indexed")
}
func (s *PackageListSuite) TestSearch(c *C) {
@@ -312,7 +312,7 @@ func (s *PackageListSuite) TestSearch(c *C) {
func (s *PackageListSuite) TestFilter(c *C) {
c.Check(func() {
_, _ = s.list.Filter(FilterOptions{
s.list.Filter(FilterOptions{
Queries: []PackageQuery{&PkgQuery{"abcd", "0.3", "i386"}},
})
}, Panics, "list not indexed, can't filter")
@@ -479,7 +479,7 @@ func (s *PackageListSuite) TestVerifyDependencies(c *C) {
{Pkg: "mail-agent", Relation: VersionDontCare, Version: "", Architecture: "arm"}})
for _, p := range s.sourcePackages {
_ = s.il.Add(p)
s.il.Add(p)
}
missing, err = s.il.VerifyDependencies(DepFollowSource, []string{"i386", "amd64"}, s.il, nil)
@@ -503,52 +503,3 @@ func (s *PackageListSuite) TestArchitectures(c *C) {
sort.Strings(archs)
c.Check(archs, DeepEquals, []string{"amd64", "arm", "i386", "s390"})
}
func (s *PackageListSuite) TestFilterLatest(c *C) {
list := NewPackageList()
older := packageStanza.Copy()
older["Version"] = "1.0"
olderPkg := NewPackageFromControlFile(older)
_ = list.Add(olderPkg)
newer := packageStanza.Copy()
newer["Version"] = "2.0"
newerPkg := NewPackageFromControlFile(newer)
_ = list.Add(newerPkg)
shared := packageStanza.Copy()
shared["Architecture"] = ArchitectureAll
shared["Version"] = "3.0"
shared["Package"] = "shared"
sharedPkg := NewPackageFromControlFile(shared)
_ = list.Add(sharedPkg)
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.Len(), Equals, 2)
c.Check(filtered.Has(newerPkg), Equals, true)
c.Check(filtered.Has(sharedPkg), Equals, true)
}
func (s *PackageListSuite) TestFilterLatestPreservesDuplicatesFlag(c *C) {
list := NewPackageListWithDuplicates(true, 2)
_ = list.Add(NewPackageFromControlFile(packageStanza.Copy()))
another := packageStanza.Copy()
another["Version"] = "7.41-1"
_ = list.Add(NewPackageFromControlFile(another))
filtered, err := list.FilterLatest()
c.Assert(err, IsNil)
c.Assert(filtered.duplicatesAllowed, Equals, true)
}
func (s *PackageListSuite) TestFilterLatestNil(c *C) {
var list *PackageList
filtered, err := list.FilterLatest()
c.Assert(err, ErrorMatches, "package list is nil")
c.Assert(filtered, IsNil)
}
+7 -8
View File
@@ -69,7 +69,7 @@ func (repo *LocalRepo) Encode() []byte {
var buf bytes.Buffer
encoder := codec.NewEncoder(&buf, &codec.MsgpackHandle{})
_ = encoder.Encode(repo)
encoder.Encode(repo)
return buf.Bytes()
}
@@ -116,7 +116,7 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
return result
}
_ = collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
r := &LocalRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding local repo: %s\n", err)
@@ -159,17 +159,15 @@ func (collection *LocalRepoCollection) Add(repo *LocalRepo) error {
// Update stores updated information about repo in DB
func (collection *LocalRepoCollection) Update(repo *LocalRepo) error {
batch := collection.db.CreateBatch()
_ = batch.Put(repo.Key(), repo.Encode())
batch.Put(repo.Key(), repo.Encode())
if repo.packageRefs != nil {
_ = batch.Put(repo.RefKey(), repo.packageRefs.Encode())
batch.Put(repo.RefKey(), repo.packageRefs.Encode())
}
return batch.Write()
}
// LoadComplete loads additional information for local repo
func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
repo.packageRefs = &PackageRefList{}
encoded, err := collection.db.Get(repo.RefKey())
if err == database.ErrNotFound {
return nil
@@ -178,6 +176,7 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error {
return err
}
repo.packageRefs = &PackageRefList{}
return repo.packageRefs.Decode(encoded)
}
@@ -248,7 +247,7 @@ func (collection *LocalRepoCollection) Drop(repo *LocalRepo) error {
delete(collection.cache, repo.UUID)
batch := collection.db.CreateBatch()
_ = batch.Delete(repo.Key())
_ = batch.Delete(repo.RefKey())
batch.Delete(repo.Key())
batch.Delete(repo.RefKey())
return batch.Write()
}
+10 -22
View File
@@ -21,8 +21,8 @@ var _ = Suite(&LocalRepoSuite{})
func (s *LocalRepoSuite) SetUpTest(c *C) {
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.list = NewPackageList()
_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.reflist = NewPackageRefListFromPackageList(s.list)
@@ -31,7 +31,7 @@ func (s *LocalRepoSuite) SetUpTest(c *C) {
}
func (s *LocalRepoSuite) TearDownTest(c *C) {
_ = s.db.Close()
s.db.Close()
}
func (s *LocalRepoSuite) TestString(c *C) {
@@ -88,14 +88,14 @@ func (s *LocalRepoCollectionSuite) SetUpTest(c *C) {
s.collection = NewLocalRepoCollection(s.db)
s.list = NewPackageList()
_ = s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
_ = s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.list.Add(&Package{Name: "lib", Version: "1.7", Architecture: "i386"})
s.list.Add(&Package{Name: "app", Version: "1.9", Architecture: "amd64"})
s.reflist = NewPackageRefListFromPackageList(s.list)
}
func (s *LocalRepoCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close()
s.db.Close()
}
func (s *LocalRepoCollectionSuite) TestAddByName(c *C) {
@@ -133,18 +133,6 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) {
c.Assert(r.String(), Equals, repo.String())
}
func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) {
repo := NewLocalRepo("local1", "Comment 1")
c.Assert(s.collection.Update(repo), IsNil)
r, err := s.collection.ByName("local1")
c.Assert(err, IsNil)
c.Assert(s.collection.LoadComplete(r), IsNil)
c.Assert(r.packageRefs, NotNil)
c.Assert(r.NumPackages(), Equals, 0)
}
func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
repo := NewLocalRepo("local1", "Comment 1")
c.Assert(s.collection.Update(repo), IsNil)
@@ -168,7 +156,7 @@ func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
repo := NewLocalRepo("local1", "Comment 1")
_ = s.collection.Add(repo)
s.collection.Add(repo)
count := 0
err := s.collection.ForEach(func(*LocalRepo) error {
@@ -190,10 +178,10 @@ func (s *LocalRepoCollectionSuite) TestForEachAndLen(c *C) {
func (s *LocalRepoCollectionSuite) TestDrop(c *C) {
repo1 := NewLocalRepo("local1", "Comment 1")
_ = s.collection.Add(repo1)
s.collection.Add(repo1)
repo2 := NewLocalRepo("local2", "Comment 2")
_ = s.collection.Add(repo2)
s.collection.Add(repo2)
r1, _ := s.collection.ByUUID(repo1.UUID)
c.Check(r1, Equals, repo1)
@@ -220,6 +208,6 @@ func (s *LocalRepoCollectionSuite) TestDropNonExisting(c *C) {
_, err := s.collection.ByUUID(repo.UUID)
c.Check(err, ErrorMatches, "local repo .* not found")
_ = s.collection.Drop(repo)
err = s.collection.Drop(repo)
c.Check(s.collection.Drop(repo), ErrorMatches, "local repo not found")
}
+1 -1
View File
@@ -565,7 +565,7 @@ func (p *Package) CalculateContents(packagePool aptly.PackagePool, progress aptl
}
return nil, err
}
defer func() { _ = reader.Close() }()
defer reader.Close()
contents, err := GetContentsFromDeb(reader, file.Filename)
if err != nil {
+2 -2
View File
@@ -309,7 +309,7 @@ func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList)
}
if q.Matches(pkg) {
_ = result.Add(pkg)
result.Add(pkg)
}
}
@@ -337,7 +337,7 @@ func (collection *PackageCollection) SearchByKey(arch, name, version string) (re
}
if pkg.Architecture == arch && pkg.Name == name && pkg.Version == version {
_ = result.Add(pkg)
result.Add(pkg)
}
}
+2 -2
View File
@@ -23,7 +23,7 @@ func (s *PackageCollectionSuite) SetUpTest(c *C) {
}
func (s *PackageCollectionSuite) TearDownTest(c *C) {
_ = s.db.Close()
s.db.Close()
}
func (s *PackageCollectionSuite) TestUpdate(c *C) {
@@ -67,7 +67,7 @@ func (s *PackageCollectionSuite) TestByKey(c *C) {
func (s *PackageCollectionSuite) TestByKeyOld0_3(c *C) {
key := []byte("Pi386 vmware-view-open-client 4.5.0-297975+dfsg-4+b1")
_ = s.db.Put(key, old0_3Package)
s.db.Put(key, old0_3Package)
p, err := s.collection.ByKey(key)
c.Check(err, IsNil)
+1 -1
View File
@@ -64,7 +64,7 @@ func (files PackageFiles) Hash() uint64 {
for _, f := range files {
h.Write([]byte(f.Filename))
_ = binary.Write(h, binary.BigEndian, f.Checksums.Size)
binary.Write(h, binary.BigEndian, f.Checksums.Size)
h.Write([]byte(f.Checksums.MD5))
h.Write([]byte(f.Checksums.SHA1))
h.Write([]byte(f.Checksums.SHA256))
+2 -2
View File
@@ -1,7 +1,7 @@
package deb
import (
"os"
"io/ioutil"
"path/filepath"
"github.com/aptly-dev/aptly/aptly"
@@ -39,7 +39,7 @@ func (s *PackageFilesSuite) TestVerify(c *C) {
c.Check(result, Equals, false)
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
s.files[0].PoolPath, _ = packagePool.Import(tmpFilepath, s.files[0].Filename, &s.files[0].Checksums, false, s.cs)
+5 -5
View File
@@ -2,7 +2,7 @@ package deb
import (
"bytes"
"os"
"io/ioutil"
"path/filepath"
"regexp"
@@ -59,7 +59,7 @@ func (s *PackageSuite) TestNewUdebFromPara(c *C) {
}
func (s *PackageSuite) TestNewInstallerFromPara(c *C) {
repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false, false)
repo, _ := NewRemoteRepo("yandex", "http://example.com/debian", "squeeze", []string{"main"}, []string{}, false, false, false)
downloader := http.NewFakeDownloader()
downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/MANIFEST.udebs", "MANIFEST.udebs")
downloader.ExpectResponse("http://example.com/debian/dists/squeeze/main/installer-i386/current/images/udeb.list", "udeb.list")
@@ -395,7 +395,7 @@ func (s *PackageSuite) TestLinkFromPool(c *C) {
p := NewPackageFromControlFile(s.stanza)
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, nil, 0777), IsNil)
c.Assert(ioutil.WriteFile(tmpFilepath, nil, 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
@@ -434,7 +434,7 @@ func (s *PackageSuite) TestDownloadList(c *C) {
})
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)
list, err = p.DownloadList(packagePool, cs)
@@ -449,7 +449,7 @@ func (s *PackageSuite) TestVerifyFiles(c *C) {
cs := files.NewMockChecksumStorage()
tmpFilepath := filepath.Join(c.MkDir(), "file")
c.Assert(os.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
c.Assert(ioutil.WriteFile(tmpFilepath, []byte("abcde"), 0777), IsNil)
p.Files()[0].PoolPath, _ = packagePool.Import(tmpFilepath, p.Files()[0].Filename, &p.Files()[0].Checksums, false, cs)

Some files were not shown because too many files have changed in this diff Show More