vincent/aptly
1
0
Fork 0
mirror of https://github.com/aptly-dev/aptly.git synced 2026-06-17 07:20:20 +00:00
Code Issues Projects Releases Wiki Activity

Merge tag 'upstream/1.6.0+ds1_alpha1' into debian/master

Browse Source
This commit is contained in:
Sébastien Delafond
2024-10-15 11:05:42 +02:00
parent a1fd350573 ae0fa20aa6
commit 25dfd98672
394 changed files with 10496 additions and 4309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+6
View File
@@ -0,0 +1,6 @@
.go/
.git/
obj-x86_64-linux-gnu/
unit.out
aptly.test
build/
.flake8
+2 -2
View File
@@ -1,7 +1,7 @@
[flake8]
max-line-length = 200
max-line-length = 240
ignore = E126,E241,E741,W504
include =
system
exclude =
system/env
system/env
.github/FUNDING.yml
+1
View File
@@ -0,0 +1 @@
github: aptly-dev
.github/workflows/ci.yml
+224 -89
View File
@@ -1,5 +1,3 @@
# Based on https://github.com/aptly-dev/aptly/blob/master/.travis.yml
name: CI
on:
@@ -13,32 +11,17 @@ on:
defaults:
run:
# see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell
shell: bash --noprofile --norc -eo pipefail -x {0}
shell: bash --noprofile --norc -eo pipefail {0}
env:
DEBIAN_FRONTEND: noninteractive
jobs:
build:
name: test
runs-on: ubuntu-20.04
continue-on-error: ${{ matrix.allow_failure }}
test:
name: "Test (Ubuntu 22.04)"
runs-on: ubuntu-22.04
continue-on-error: false
timeout-minutes: 30
strategy:
fail-fast: true
matrix:
go: [1.16, 1.17, 1.18]
run_long_tests: [no]
allow_failure: [false]
include:
# Disable this for now as it's not clear how to select the latest master branch
# version of go using actions/setup-go@v2.
# - go: master
# run_long_tests: no
# allow_failure: true
- go: 1.17
run_long_tests: yes
allow_failure: false
env:
NO_FTP_ACCESS: yes
@@ -47,101 +30,253 @@ jobs:
GOPROXY: "https://proxy.golang.org"
steps:
- name: Checkout repository
uses: actions/checkout@v2
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ matrix.go }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
with:
version: v1.45.0
- name: Setup Python
uses: actions/setup-python@v2
with:
python-version: 3.9
- name: Install O/S packages
- name: "Install packages"
run: |
sudo apt-get update
sudo apt-get install -y graphviz gnupg1 gnupg2 gpgv1 gpgv2 git gcc make
sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8
- name: Install Python packages
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Run flake8"
run: |
pip install six packaging appdirs virtualenv
pip install -U pip setuptools
pip install -r system/requirements.txt
make flake8
- name: Install Azurite
- name: "Read go version from go.mod"
run: |
gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
echo "Go Version: $gover"
echo "GOVER=$gover" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Install Azurite"
id: azuright
uses: potatoqualitee/azuright@v1.1
with:
directory: ${{ runner.temp }}
- name: Get aptly version
run: |
make version
- name: Make
- name: "Run Unit Tests"
env:
RUN_LONG_TESTS: ${{ matrix.run_long_tests }}
AZURE_STORAGE_ENDPOINT: "127.0.0.1:10000"
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
make
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make test
- name: Upload code coverage
if: matrix.run_long_tests
- name: "Run Benchmark"
run: |
COVERAGE_DIR=${{ runner.temp }} make bench
- name: "Run System Tests"
env:
RUN_LONG_TESTS: 'yes'
AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1"
AZURE_STORAGE_ACCOUNT: "devstoreaccount1"
AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
sudo mkdir -p /srv ; sudo chown runner /srv
COVERAGE_DIR=${{ runner.temp }} make system-test
- name: "Merge code coverage"
run: |
go install github.com/wadey/gocovmerge@latest
~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt
- name: "Upload code coverage"
uses: codecov/codecov-action@v2
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: coverage.txt
release:
name: release
needs: build
runs-on: ubuntu-20.04
ci-debian-build:
name: "Build"
needs: test
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
name: ["Debian 13/testing", "Debian 12/bookworm", "Debian 11/bullseye", "Debian 10/buster", "Ubuntu 24.04", "Ubuntu 22.04", "Ubuntu 20.04"]
arch: ["amd64", "i386" , "arm64" , "armhf"]
include:
- name: "Debian 13/testing"
suite: trixie
image: debian:trixie-slim
- name: "Debian 12/bookworm"
suite: bookworm
image: debian:bookworm-slim
- name: "Debian 11/bullseye"
suite: bullseye
image: debian:bullseye-slim
- name: "Debian 10/buster"
suite: buster
image: debian:buster-slim
- name: "Ubuntu 24.04"
suite: noble
image: ubuntu:24.04
- name: "Ubuntu 22.04"
suite: jammy
image: ubuntu:22.04
- name: "Ubuntu 20.04"
suite: focal
image: ubuntu:20.04
container:
image: ${{ matrix.image }}
env:
APT_LISTCHANGES_FRONTEND: none
DEBIAN_FRONTEND: noninteractive
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: "Install packages"
run: |
apt-get update
apt-get install -y --no-install-recommends make ca-certificates git curl build-essential devscripts dh-golang binutils-i686-linux-gnu binutils-aarch64-linux-gnu binutils-arm-linux-gnueabihf jq
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repot for `git describe` to
# work and get the nightly verion
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v2
- name: "Read go version from go.mod"
run: |
gover=$(sed -n 's/^go \(.*\)/\1/p' go.mod)
echo "Go Version: $gover"
echo "GOVER=$gover" >> $GITHUB_OUTPUT
id: goversion
- name: Make Release
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Ensure CI build"
if: github.ref == 'refs/heads/master'
run: |
echo "FORCE_CI=true" >> $GITHUB_OUTPUT
id: force_ci
- name: "Build Debian packages"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
make dpkg DEBARCH=${{ matrix.arch }}
- name: "Check aptly credentials"
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
found=no
if [ -n "$APTLY_USER" ] && [ -n "$APTLY_PASSWORD" ]; then
found=yes
fi
echo "Aptly credentials available: $found"
echo "FOUND=$found" >> $GITHUB_OUTPUT
id: aptlycreds
- name: "Publish CI release to aptly"
if: github.ref == 'refs/heads/master' && steps.aptlycreds.outputs.FOUND == 'yes'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
.github/workflows/scripts/upload-artifacts.sh ci ${{ matrix.suite }}
- name: "Publish release to aptly"
if: startsWith(github.event.ref, 'refs/tags') && steps.aptlycreds.outputs.FOUND == 'yes'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
.github/workflows/scripts/upload-artifacts.sh release ${{ matrix.suite }}
ci-binary-build:
name: "Build"
needs: test
runs-on: ubuntu-latest
strategy:
matrix:
goos: [linux, freebsd, darwin]
goarch: ["386", "amd64", "arm", "arm64"]
exclude:
- goos: darwin
goarch: 386
- goos: darwin
goarch: arm
steps:
- name: "Checkout repository"
uses: actions/checkout@v4
with:
# fetch the whole repo for `git describe` to work
fetch-depth: 0
- name: "Read go version from go.mod"
run: |
echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: "Ensure CI build"
if: github.ref == 'refs/heads/master'
run: |
echo "FORCE_CI=true" >> $GITHUB_OUTPUT
id: force_ci
- name: "Get aptly version"
env:
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
aptlyver=$(make -s version)
echo "Aptly Version: $aptlyver"
echo "VERSION=$aptlyver" >> $GITHUB_OUTPUT
id: releaseversion
- name: "Build aptly ${{ matrix.goos }}/${{ matrix.goarch }}"
env:
GOBIN: /usr/local/bin
FORCE_CI: ${{ steps.force_ci.outputs.FORCE_CI }}
run: |
make release
make binaries GOOS=${{ matrix.goos }} GOARCH=${{ matrix.goarch }}
- name: Publish nightly release to aptly
if: github.ref == 'refs/heads/master'
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh nightly
- name: Publish release to aptly
- name: "Upload Artifacts"
uses: actions/upload-artifact@v4
if: startsWith(github.event.ref, 'refs/tags')
env:
APTLY_USER: ${{ secrets.APTLY_USER }}
APTLY_PASSWORD: ${{ secrets.APTLY_PASSWORD }}
run: |
./upload-artifacts.sh release
- name: Upload artifacts to GitHub Release
if: startsWith(github.event.ref, 'refs/tags')
uses: softprops/action-gh-release@v1
with:
body: Release ${{ github.ref }} generated by the CI.
files: build/*
name: aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}
path: build/aptly_${{ steps.releaseversion.outputs.VERSION }}_${{ matrix.goos }}_${{ matrix.goarch }}.zip
compression-level: 0 # no compression
gh-release:
name: "Github Release"
runs-on: ubuntu-latest
continue-on-error: false
needs: ci-binary-build
if: startsWith(github.event.ref, 'refs/tags')
steps:
- name: "Download Artifacts"
uses: actions/download-artifact@v4
with:
path: out/
- name: "Release"
uses: softprops/action-gh-release@v2
with:
files: "out/**/aptly_*.zip"
.github/workflows/golangci-lint.yml
+72
View File
@@ -0,0 +1,72 @@
name: golangci-lint
on:
push:
branches:
- master
pull_request:
permissions:
contents: read
# Optional: allow read access to pull request. Use with `only-new-issues` option.
# pull-requests: read
jobs:
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: "Read go version from go.mod"
run: |
echo "GOVER=$(sed -n 's/^go \(.*\)/\1/p' go.mod)" >> $GITHUB_OUTPUT
id: goversion
- name: "Setup Go"
uses: actions/setup-go@v3
with:
go-version: ${{ steps.goversion.outputs.GOVER }}
- name: Create VERSION file
run: |
make -s version | tr -d '\n' > VERSION
shell: sh
- name: Install and initialize swagger
run: |
go install github.com/swaggo/swag/cmd/swag@latest
swag init -q --markdownFiles docs
shell: sh
- name: golangci-lint
uses: golangci/golangci-lint-action@v4
with:
# Require: The version of golangci-lint to use.
# When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version.
# When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit.
version: v1.54.1
# Optional: working directory, useful for monorepos
# working-directory: somedir
# Optional: golangci-lint command line arguments.
#
# Note: By default, the `.golangci.yml` file should be at the root of the repository.
# The location of the configuration file can be changed by using `--config=`
# args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true
# Optional: if set to true, then all caching functionality will be completely disabled,
# takes precedence over all other caching options.
# skip-cache: true
# Optional: if set to true, then the action won't cache or restore ~/go/pkg.
# skip-pkg-cache: true
# Optional: if set to true, then the action won't cache or restore ~/.cache/go-build.
# skip-build-cache: true
# Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'.
# install-mode: "goinstall"
.github/workflows/scripts/upload-artifacts.sh
Executable
+144
View File
@@ -0,0 +1,144 @@
#!/bin/sh
set -e
builds=build/
packages=${builds}*.deb
folder=`mktemp -u tmp.XXXXXXXXXXXXXXX`
aptly_user="$APTLY_USER"
aptly_password="$APTLY_PASSWORD"
aptly_api="https://aptly-ops.aptly.info"
version=`make version`
action=$1
dist=$2
usage() {
echo "Usage: $0 ci buster|bullseye|bookworm|focal|jammy|noble" >&2
echo " $0 release" >&2
}
# repos and publish must be created beforehand:
#!/bin/sh
#for dist in buster bullseye bookworm focal jammy noble
#do
# for build in ci release
# do
# echo
# echo "# Creating and publishing $build/$dist"
# aptly repo create -distribution=$dist -component=main aptly-$build-$dist
# aptly publish repo -multi-dist -architectures="amd64,i386,arm64,armhf" -acquire-by-hash -component=main \
# -distribution=$dist -batch -keyring=aptly.pub \
# aptly-$build-$dist \
# s3:repo.aptly.info:$build
# done
#done
if [ -z "$action" ]; then
usage
exit 1
fi
if [ "action" = "ci" ] && [ -z "$dist" ]; then
usage
exit 1
fi
if [ -z "$aptly_user" ] || [ -z "$aptly_password" ]; then
usage
echo Error: please set APTLY_USER and APTLY_PASSWORD
exit 1
fi
echo "Publishing version '$version' to $action for $dist...\n"
upload()
{
echo "\nUploading files:"
for file in $packages; do
echo " - $file"
jsonret=`curl -fsS -X POST -F "file=@$file" -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
done
}
cleanup() {
echo "\nCleanup..."
jsonret=`curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/files/$folder`
}
trap cleanup EXIT
update_publish() {
_publish=$1
_dist=$2
jsonret=`curl -fsS -X PUT -H 'Content-Type: application/json' --data \
'{"AcquireByHash": true, "MultiDist": true,
"Signing": {"Batch": true, "Keyring": "aptly.repo/aptly.pub", "secretKeyring": "aptly.repo/aptly.sec", "PassphraseFile": "aptly.repo/passphrase"}}' \
-u $aptly_user:$aptly_password ${aptly_api}/api/publish/$_publish/$_dist?_async=true`
_task_id=`echo $jsonret | jq .ID`
_success=0
for t in `seq 180`
do
jsonret=`curl -fsS -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_task_id`
_state=`echo $jsonret | jq .State`
if [ "$_state" = "2" ]; then
_success=1
curl -fsS -X DELETE -u $aptly_user:$aptly_password ${aptly_api}/api/tasks/$_task_id
break
fi
if [ "$_state" = "3" ]; then
echo Error: publish failed
exit 1
fi
sleep 1
done
if [ "$_success" -ne 1 ]; then
echo "Error: publish failed (timeout)"
exit 1
fi
}
if [ "$action" = "ci" ]; then
if echo "$version" | grep -vq "+"; then
# skip ci when on release tag
exit 0
fi
aptly_repository=aptly-ci-$dist
aptly_published=s3:repo.aptly.info:ci
elif [ "$action" = "release" ]; then
aptly_repository=aptly-release-$dist
aptly_published=s3:repo.aptly.info:release
fi
upload
echo "\nAdding packages to $aptly_repository ..."
jsonret=`curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder`
echo "\nUpdating published repo $aptly_published ..."
update_publish $aptly_published $dist
# if [ "$action" = "OBSOLETErelease" ]; then
# aptly_repository=aptly-release
# aptly_snapshot=aptly-$version
# aptly_published=s3:repo.aptly.info:./squeeze
#
# echo "\nAdding packages to $aptly_repository..."
# curl -fsS -X POST -u $aptly_user:$aptly_password ${aptly_api}/api/repos/$aptly_repository/file/$folder
# echo
#
# echo "\nCreating snapshot $aptly_snapshot from repo $aptly_repository..."
# curl -fsS -X POST -u $aptly_user:$aptly_password -H 'Content-Type: application/json' --data \
# "{\"Name\":\"$aptly_snapshot\"}" ${aptly_api}/api/repos/$aptly_repository/snapshots
# echo
#
# echo "\nSwitching published repo $aptly_published to use snapshot $aptly_snapshot..."
# curl -fsS -X PUT -H 'Content-Type: application/json' --data \
# "{\"AcquireByHash\": true, \"Snapshots\": [{\"Component\": \"main\", \"Name\": \"$aptly_snapshot\"}],
# \"Signing\": {\"Batch\": true, \"Keyring\": \"aptly.repo/aptly.pub\",
# \"secretKeyring\": \"aptly.repo/aptly.sec\", \"PassphraseFile\": \"aptly.repo/passphrase\"}}" \
# -u $aptly_user:$aptly_password ${aptly_api}/api/publish/$aptly_published
# echo
# fi
.gitignore
+31 -5
View File
@@ -2,11 +2,14 @@
*.o
*.a
*.so
unit.out
# Folders
_obj
_test
tmp/
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
@@ -32,13 +35,36 @@ root/
man/aptly.1.html
man/aptly.1.ronn
.goxc.local.json
system/env/
# created by make build for release artifacts
VERSION
aptly.test
build/
pgp/keyrings/aptly2*.gpg
pgp/keyrings/aptly2*.gpg~
pgp/keyrings/.#*
system/files/aptly2.gpg~
system/files/aptly2_passphrase.gpg~
*.creds
.go/
obj-x86_64-linux-gnu/
# debian
debian/.debhelper/
debian/aptly.debhelper.log
debian/aptly.postrm.debhelper
debian/aptly.substvars
debian/aptly/
debian/debhelper-build-stamp
debian/files
debian/aptly-api/
debian/aptly-api.debhelper.log
debian/aptly-api.postrm.debhelper
debian/aptly-api.substvars
debian/aptly-dbg.debhelper.log
debian/aptly-dbg.substvars
debian/aptly-dbg/
docs/
.golangci.yml
-3
View File
@@ -5,7 +5,6 @@ run:
linters:
disable-all: true
enable:
- deadcode
- goconst
- gofmt
- goimports
@@ -14,6 +13,4 @@ linters:
- misspell
- revive
- staticcheck
- structcheck
- varcheck
- vetshadow
.goxc.json
-45
View File
@@ -1,45 +0,0 @@
{
"AppName": "aptly",
"ArtifactsDest": "xc-out/",
"TasksExclude": [
"rmbin",
"go-test",
"go-vet"
],
"TasksAppend": [
"bintray"
],
"TaskSettings": {
"debs": {
"metadata": {
"maintainer": "Andrey Smirnov",
"maintainer-email": "me@smira.ru",
"description": "Debian repository management tool"
},
"metadata-deb": {
"License": "MIT",
"Homepage": "https://www.aptly.info/",
"Depends": "bzip2, xz-utils, gnupg, gpgv",
"Suggests": "graphviz"
},
"other-mapped-files": {
"/": "root/"
}
},
"bintray": {
"repository": "aptly",
"subject": "smira",
"package": "aptly",
"downloadspage": "bintray.md"
}
},
"ResourcesInclude": "README.rst,LICENSE,AUTHORS,man/aptly.1",
"BuildConstraints": "linux,386 linux,amd64 darwin,amd64 freebsd,386 freebsd,amd64",
"MainDirsExclude": "_man,vendor",
"BuildSettings": {
"LdFlagsXVars": {
"Version": "main.Version"
}
},
"ConfigVersion": "0.9"
}
.travis.yml
-106
View File
@@ -1,106 +0,0 @@
dist: xenial
sudo: required
language: go
go_import_path: github.com/aptly-dev/aptly
addons:
apt:
packages:
- python-virtualenv
- graphviz
- gnupg2
- gpgv2
env:
global:
- secure: "EcCzJsqQ3HnIkprBPS1YHErsETcb7KQFBYEzVDE7RYDApWeapLq+r/twMtWMd/fkGeLzr3kWSg7nhSadeHMLYeMl9j+U7ncC5CWG5NMBOj/jowlb9cMCCDlmzMoZLAgR6jm1cJyrWCLsWVlv+D0ZiB0fx4xaBZP/gIr9g6nEwC8="
- secure: "OxiVNmre2JzUszwPNNilKDgIqtfX2gnRSsVz6nuySB1uO2yQsOQmKWJ9cVYgH2IB5H8eWXKOhexcSE28kz6TPLRuEcU9fnqKY3uEkdwm7rJfz9lf+7C4bJEUdA1OIzJppjnWUiXxD7CEPL1DlnMZM24eDQYqa/4WKACAgkK53gE="
- NO_FTP_ACCESS: "yes"
- BOTO_CONFIG: /dev/null
- GO111MODULE: "on"
- GOPROXY: https://proxy.golang.org
matrix:
allow_failures:
- go: master
env: RUN_LONG_TESTS=no
fast_finish: true
include:
- go: 1.11.x
env: RUN_LONG_TESTS=no
- go: 1.12.x
env: RUN_LONG_TESTS=no
- go: 1.13.x
env: RUN_LONG_TESTS=no
- go: 1.14.x
env: RUN_LONG_TESTS=yes
- go: 1.15.x
env:
- RUN_LONG_TESTS=yes
- DEPLOY_BINARIES=yes
- APTLY_USER=aptly
- secure: "ejVss+Ansvk9f237iXVd87KA8N/SkfJkEdr/KCw9WRkVw3M9WyYtFnqpakIUPFT8RsSc7MW+RU4OM90DsbE9dbDIL0oW+t6QH/IfGjNG2HjDiGEWN/tMLeAQTtzPaVqlItJBo0ILMF2K6NrgkYBYU+tZ8gk5w7CuARvAk82d00o="
- go: master
env: RUN_LONG_TESTS=no
before_install:
- virtualenv system/env
- . system/env/bin/activate
- pip install six packaging appdirs
- pip install -U pip setuptools
- pip install -r system/requirements.txt
- make version
install:
- make prepare
after_success:
- bash <(curl -s https://codecov.io/bash)
notifications:
webhooks:
urls:
- "https://webhooks.gitter.im/e/c691da114a41eed6ec45"
on_success: change # options: [always|never|change] default: always
on_failure: always # options: [always|never|change] default: always
on_start: false # default: false
before_deploy:
- make release
deploy:
- provider: releases
api_key:
secure: XHxYAFBzzgOZyK6JXQpEp0kGrZPmd02esEJjwJXZpWT68kRzCCrBXg+x3vIcgRtl82oQbflv/ThNlGT80iqSmd+Itsa5lUJoJnRxbP8qSykfCXmkrgsHIxbGxWIL+JHAWmwQdkV91kDS04nmjl9MDptLId0tuleWwcMH6h1hgMg=
file_glob: true
file: build/*
skip_cleanup: true
on:
tags: true
condition: "$DEPLOY_BINARIES = yes"
- provider: s3
access_key_id:
secure: "I2etn22HHsQjJNhr6zdM/P4VLCYwEA/6HEf2eGvwey93oLeog+KnDCUI7lwGAHYuwzyDGQbZZ6YdoNc3b0kxaRWT0W+ke78TAdJhTZ+xbqGfEWv1er0zklJLOsimYF097rDJw8g3Oh/Gjwt5TTp0GJ5l3IhJ6zepNsKCMuwQpJM="
secret_access_key:
secure: "inRWX7FuyhkhKzGknSd2/mjZaNFZm/zHMejM99OF6PiGLNtyt/esdA0ToYL8B8Icl0/SISlLlEr/DDa4OGENKueFVeHrKH7OK0jVbWp9Yvw4hCXSlw9VmlkHDMQrC4gybS2Hf7el8N4AFVqyeUE7LqiP3WruHRdbE9XgOnTkLkg="
bucket: aptly-nightly
skip_cleanup: true
acl: public-read
local_dir: build
on:
branch: master
condition: "$DEPLOY_BINARIES = yes"
- provider: script
script: bash upload-artifacts.sh nightly
skip_cleanup: true
on:
branch: master
condition: "$DEPLOY_BINARIES = yes"
- provider: script
script: bash upload-artifacts.sh release
skip_cleanup: true
on:
tags: true
condition: "$DEPLOY_BINARIES = yes"
AUTHORS
+14
View File
@@ -49,3 +49,17 @@ List of contributors, in chronological order:
* Samuel Mutel (https://github.com/smutel)
* Russell Greene (https://github.com/russelltg)
* Wade Simmons (https://github.com/wadey)
* Steven Stone (https://github.com/smstone)
* Josh Bayfield (https://github.com/jbayfield)
* Boxjan (https://github.com/boxjan)
* Mauro Regli (https://github.com/reglim)
* Alexander Zubarev (https://github.com/strike)
* Nicolas Dostert (https://github.com/acdn-ndostert)
* Ryan Gonzalez (https://github.com/refi64)
* Paul Cacheux (https://github.com/paulcacheux)
* Nic Waller (https://github.com/sf-nwaller)
* iofq (https://github.com/iofq)
* Noa Resare (https://github.com/nresare)
* Ramón N.Rodriguez (https://github.com/runitonmetal)
* Golf Hu (https://github.com/hudeng-go)
* Cookie Fei (https://github.com/wuhuang26)
CODE_OF_CONDUCT.md
+1 -1
View File
@@ -55,7 +55,7 @@ further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at [team@aptly.info](mailto:team@aptly.info). All
reported by contacting the project team on [Aptly Discussions](https://github.com/aptly-dev/aptly/discussions). All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
CONTRIBUTING.md
+96 -45
View File
@@ -2,7 +2,7 @@
:+1::tada: First off, thanks for taking the time to contribute! :tada::+1:
The following is a set of guidelines for contributing to [aptly](https://github.com/smira/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
The following is a set of guidelines for contributing to [aptly](https://github.com/aptly-dev/aplty) and related repositories, which are hosted in the [aptly-dev Organization](https://github.com/aptly-dev) on GitHub.
These are just guidelines, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
## What should I know before I get started?
@@ -11,7 +11,7 @@ These are just guidelines, not rules. Use your best judgment, and feel free to p
This project adheres to the Contributor Covenant [code of conduct](CODE_OF_CONDUCT.md).
By participating, you are expected to uphold this code.
Please report unacceptable behavior to [team@aptly.info](mailto:team@aptly.info).
Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discussions](https://github.com/aptly-dev/aptly/discussions)
### List of Repositories
@@ -60,7 +60,7 @@ If you want to update website, please follow steps below:
We're always looking for new contributions to [FAQ](https://www.aptly.info/doc/faq/), [tutorials](https://www.aptly.info/tutorial/),
general fixes, clarifications, misspellings, grammar mistakes!
### Your First Code Contribution
### Code Contribution
Please follow [next section](#development-setup) on development process. When change is ready, please submit PR
following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
@@ -68,20 +68,6 @@ following [PR template](.github/PULL_REQUEST_TEMPLATE.md).
Make sure that purpose of your change is clear, all the tests and checks pass, and all new code is covered with tests
if that is possible.
## Development Setup
This section describes local setup to start contributing to aptly source.
### Go & Python
You would need `Go` (latest version is recommended) and `Python` 2.7.x (3.x is not supported yet).
If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
to prepend it or to skip this test if you're security conscious.
### Forking and Cloning
As aptly is using Go modules, aptly repository could be cloned to any location on the file system:
@@ -98,7 +84,94 @@ to specify your remote name when pushing branches:
git push <user> <your-branch>
### Dependencies
## Development Setup
Working on aptly code can be done locally on the development machine, or for convenience by using docker. The next sections describe the setup process.
### Docker Development Setup
This section describes the docker setup to start contributing to aptly.
#### Dependencies
Install the following on your development machine:
- docker
- make
- git
#### Create docker container
To build the development docker image, run:
```
make docker-image
```
#### Build aptly
To build the aptly in the development docker container, run:
```
make docker-build
```
#### Running aptly commands
To run aptly commands in the development docker container, run:
```
make docker-aptly
```
Example:
```
$ make docker-aptly
bash: cannot set terminal process group (16): Inappropriate ioctl for device
bash: no job control in this shell
aptly@b43e8473ef81:/app$ aptly version
aptly version: 1.5.0+189+g0fc90dff
```
#### Running unit tests
In order to run aptly unit tests, enter the following:
```
make docker-unit-tests
```
#### Running system tests
In order to run aptly system tests, enter the following:
```
make docker-system-tests
```
#### Running golangci-lint
In order to run aptly unit tests, run:
```
make docker-lint
```
#### More info
Run `make help` for more information.
### Local Development Setup
This section describes local setup to start contributing to aptly.
#### Go & Python
You would need `Go` (latest version is recommended) and `Python` 3.9 (or newer, the CI currently tests against 3.11).
If you're new to Go, follow [getting started guide](https://golang.org/doc/install) to install it and perform
initial setup. With Go 1.8+, default `$GOPATH` is `$HOME/go`, so rest of this document assumes that.
Usually `$GOPATH/bin` is appended to your `$PATH` to make it easier to run built binaries, but you might choose
to prepend it or to skip this test if you're security conscious.
#### Dependencies
You would need some additional tools and Python virtual environment to run tests and checks, install them with:
@@ -110,7 +183,7 @@ Aptly is using Go modules to manage dependencies, download modules using:
make modules
### Building
#### Building
If you want to build aptly binary from your current source tree, run:
@@ -124,7 +197,7 @@ Or, if it's not on your path:
~/go/bin/aptly
### Unit-tests
#### Unit-tests
aptly has two kinds of tests: unit-tests and functional (system) tests. Functional tests are preferred way to test any
feature, but some features are much easier to test with unit-tests (e.g. algorithms, failure scenarios, ...)
@@ -133,7 +206,7 @@ aptly is using standard Go unit-test infrastructure plus [gocheck](http://labix.
make test
### Functional Tests
#### Functional Tests
Functional tests are implemented in Python, and they use custom test runner which is similar to Python unit-test
runner. Most of the tests start with clean aptly state, run some aptly commands to prepare environment, and finally
@@ -180,7 +253,7 @@ There are some packages available under `system/files/` directory which are used
this default location. You can run aptly under different user or by using non-default config location with non-default
aptly root directory.
### Style Checks
#### Style Checks
Style checks could be run with:
@@ -191,7 +264,7 @@ for the linter could be found in [.golangci.yml](.golangci.yml) file.
Python code (system tests) are linted with [flake8 tool](https://pypi.python.org/pypi/flake8).
### Vendored Code
#### Vendored Code
aptly is using Go vendoring for all the libraries aptly depends upon. `vendor/` directory is checked into the source
repository to avoid any problems if source repositories go away. Go build process will automatically prefer vendored
@@ -217,25 +290,3 @@ Bash and Zsh completion for aptly reside in the same repo under in [completion.d
When new option or command is introduced, bash completion should be updated to reflect that change.
When aptly package is being built, it automatically pulls bash completion and man page into the package.
## Design
This section requires future work.
*TBD*
### Database
### Package Pool
### Package
### PackageList, PackageRefList
### LocalRepo, RemoteRepo, Snapshot
### PublishedRepository
### Context
### Collections, CollectionFactory
Makefile
+162 -55
View File
@@ -1,81 +1,188 @@
GOVERSION=$(shell go version | awk '{print $$3;}')
ifdef TRAVIS_TAG
TAG=$(TRAVIS_TAG)
else
TAG="$(shell git describe --tags --always)"
endif
VERSION=$(shell echo $(TAG) | sed 's@^v@@' | sed 's@-@+@g')
PACKAGES=context database deb files gpg http query swift s3 utils
GOPATH=$(shell go env GOPATH)
VERSION=$(shell make -s version)
PYTHON?=python3
TESTS?=
BINPATH?=$(GOPATH)/bin
RUN_LONG_TESTS?=yes
GOLANGCI_LINT_VERSION=v1.54.1 # version supporting go 1.19
COVERAGE_DIR?=$(shell mktemp -d)
GOOS=$(shell go env GOHOSTOS)
GOARCH=$(shell go env GOHOSTARCH)
all: modules test bench check system-test
# Uncomment to update test outputs
# CAPTURE := "--capture"
prepare:
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.43.0
help: ## Print this help
@grep -E '^[a-zA-Z][a-zA-Z0-9_-]*:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
modules:
go mod download
prepare: ## Install go module dependencies
# Prepare go modules
go mod verify
go mod tidy -v
# Generate VERSION file
go generate
dev:
go get -u github.com/laher/goxc
releasetype: # Print release type: ci (on any branch/commit), release (on a tag)
@reltype=ci ; \
gitbranch=`git rev-parse --abbrev-ref HEAD` ; \
if [ "$$gitbranch" = "HEAD" ] && [ "$$FORCE_CI" != "true" ]; then \
gittag=`git describe --tags --exact-match 2>/dev/null` ;\
if echo "$$gittag" | grep -q '^v[0-9]'; then \
reltype=release ; \
fi ; \
fi ; \
echo $$reltype
check: system/env
ifeq ($(RUN_LONG_TESTS), yes)
golangci-lint run
system/env/bin/flake8
endif
version: ## Print aptly version
@ci="" ; \
if [ "`make -s releasetype`" = "ci" ]; then \
ci=`TZ=UTC git show -s --format='+%cd.%h' --date=format-local:'%Y%m%d%H%M%S'`; \
fi ; \
if which dpkg-parsechangelog > /dev/null 2>&1; then \
echo `dpkg-parsechangelog -S Version`$$ci; \
else \
echo `grep ^aptly -m1 debian/changelog | sed 's/.*(\([^)]\+\)).*/\1/'`$$ci ; \
fi
swagger-install:
# Install swag
@test -f $(BINPATH)/swag || GOOS=linux GOARCH=amd64 go install github.com/swaggo/swag/cmd/swag@latest
swagger: swagger-install
# Generate swagger docs
@PATH=$(BINPATH)/:$(PATH) swag init --markdownFiles docs
etcd-install:
# Install etcd
test -d /srv/etcd || system/t13_etcd/install-etcd.sh
flake8: ## run flake8 on system test python files
flake8 system/
lint:
# Install golangci-lint
go install github.com/golangci/golangci-lint/cmd/golangci-lint@$(GOLANGCI_LINT_VERSION)
# Running lint
@PATH=$(BINPATH)/:$(PATH) golangci-lint run
build: prepare swagger ## Build aptly
go build -o build/aptly
install:
go install -v -ldflags "-X main.Version=$(VERSION)"
@echo "\e[33m\e[1mBuilding aptly ...\e[0m"
go generate
@out=`mktemp`; if ! go install -v > $$out 2>&1; then cat $$out; rm -f $$out; echo "\nBuild failed\n"; exit 1; else rm -f $$out; fi
system/env: system/requirements.txt
ifeq ($(RUN_LONG_TESTS), yes)
rm -rf system/env
$(PYTHON) -m venv system/env
system/env/bin/pip install -r system/requirements.txt
endif
test: prepare swagger etcd-install ## Run unit tests
@echo "\e[33m\e[1mStarting etcd ...\e[0m"
@mkdir -p /tmp/etcd-data; system/t13_etcd/start-etcd.sh > /tmp/etcd-data/etcd.log 2>&1 &
@echo "\e[33m\e[1mRunning go test ...\e[0m"
go test -v ./... -gocheck.v=true -coverprofile=unit.out; echo $$? > .unit-test.ret
@echo "\e[33m\e[1mStopping etcd ...\e[0m"
@pid=`cat /tmp/etcd.pid`; kill $$pid
@rm -f /tmp/etcd-data/etcd.log
@ret=`cat .unit-test.ret`; if [ "$$ret" = "0" ]; then echo "\n\e[32m\e[1mUnit Tests SUCCESSFUL\e[0m"; else echo "\n\e[31m\e[1mUnit Tests FAILED\e[0m"; fi; rm -f .unit-test.ret; exit $$ret
system-test: install system/env
ifeq ($(RUN_LONG_TESTS), yes)
system-test: prepare swagger etcd-install ## Run system tests
# build coverage binary
go test -v -coverpkg="./..." -c -tags testruncli
# Download fixture-db, fixture-pool, etcd.db
if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi
if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi
PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS)
endif
test:
go test -v ./... -gocheck.v=true -race -coverprofile=coverage.txt -covermode=atomic
test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz)
# Run system tests
PATH=$(BINPATH)/:$(PATH) && FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)
bench:
@echo "\e[33m\e[1mRunning benchmark ...\e[0m"
go test -v ./deb -run=nothing -bench=. -benchmem
serve: prepare swagger-install ## Run development server (auto recompiling)
test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3
cp debian/aptly.conf ~/.aptly.conf
sed -i /enableSwaggerEndpoint/s/false/true/ ~/.aptly.conf
PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,cmd,systemd -- api serve -listen 0.0.0.0:3142
dpkg: prepare swagger ## Build debian packages
@test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1)
# set debian version
@if [ "`make -s releasetype`" = "ci" ]; then \
echo CI Build, setting version... ; \
cp debian/changelog debian/changelog.dpkg-bak ; \
DEBEMAIL="CI <ci@aptly>" dch -v `make -s version` "CI build" ; \
fi
# Run dpkg-buildpackage
buildtype="any" ; \
if [ "$(DEBARCH)" = "amd64" ]; then \
buildtype="any,all" ; \
fi ; \
echo "\e[33m\e[1mBuilding: $$buildtype\e[0m" ; \
dpkg-buildpackage -us -uc --build=$$buildtype -d --host-arch=$(DEBARCH)
# cleanup
@test -f debian/changelog.dpkg-bak && mv debian/changelog.dpkg-bak debian/changelog || true ; \
mkdir -p build && mv ../*.deb build/ ; \
cd build && ls -l *.deb
binaries: prepare swagger ## Build binary releases (FreeBSD, MacOS, Linux tar)
# build aptly
GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o build/tmp/aptly -ldflags='-extldflags=-static'
# install
@mkdir -p build/tmp/man build/tmp/completion/bash_completion.d build/tmp/completion/zsh/vendor-completions
@cp man/aptly.1 build/tmp/man/
@cp completion.d/aptly build/tmp/completion/bash_completion.d/
@cp completion.d/_aptly build/tmp/completion/zsh/vendor-completions/
@cp README.rst LICENSE AUTHORS build/tmp/
@gzip -f build/tmp/man/aptly.1
@path="aptly_$(VERSION)_$(GOOS)_$(GOARCH)"; \
rm -rf "build/$$path"; \
mv build/tmp build/"$$path"; \
rm -rf build/tmp; \
cd build; \
zip -r "$$path".zip "$$path" > /dev/null \
&& echo "Built build/$${path}.zip"; \
rm -rf "$$path"
docker-image: ## Build aptly-dev docker image
@docker build -f system/Dockerfile . -t aptly-dev
docker-build: ## Build aptly in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build
docker-shell: ## Run aptly and other commands in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true
docker-deb: ## Build debian packages in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64
docker-unit-test: ## Run unit tests in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper test
docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests)
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper system-test TEST=$(TEST)
docker-serve: ## Run development server (auto recompiling) on http://localhost:3142
@docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true
docker-lint: ## Run golangci-lint in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint
docker-binaries: ## Build binary releases (FreeBSD, MacOS, Linux tar) in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries
docker-man: ## Create man page in docker container
@docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man
mem.png: mem.dat mem.gp
gnuplot mem.gp
open mem.png
goxc: dev
rm -rf root/
mkdir -p root/usr/share/man/man1/ root/etc/bash_completion.d/ root/usr/share/zsh/vendor-completions/
cp man/aptly.1 root/usr/share/man/man1
cp completion.d/aptly root/etc/bash_completion.d/
cp completion.d/_aptly root/usr/share/zsh/vendor-completions/
gzip root/usr/share/man/man1/aptly.1
goxc -pv=$(VERSION) -max-processors=2 $(GOXC_OPTS)
release: GOXC_OPTS=-tasks-=bintray,go-vet,go-test,rmbin
release: goxc
rm -rf build/
mkdir -p build/
mv xc-out/$(VERSION)/aptly_$(VERSION)_* build/
man:
man: ## Create man pages
make -C man
version:
@echo $(VERSION)
clean: ## remove local build and module cache
# Clean all generated and build files
test -d .go/ && chmod u+w -R .go/ && rm -rf .go/ || true
rm -rf build/ obj-*-linux-gnu* tmp/
rm -f unit.out aptly.test VERSION docs/docs.go docs/swagger.json docs/swagger.yaml docs/swagger.conf
find system/ -type d -name __pycache__ -exec rm -rf {} \; 2>/dev/null || true
.PHONY: man modules version release goxc
.PHONY: help man prepare swagger version binaries docker-release docker-system-test docker-unit-test docker-lint docker-build docker-image build docker-shell clean releasetype dpkg serve docker-serve flake8
README.rst
+1 -1
View File
@@ -9,7 +9,7 @@ aptly
:target: https://codecov.io/gh/aptly-dev/aptly
.. image:: https://badges.gitter.im/Join Chat.svg
:target: https://gitter.im/aptly-dev/aptly?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
:target: https://matrix.to/#/#aptly:gitter.im
.. image:: https://goreportcard.com/badge/github.com/aptly-dev/aptly
:target: https://goreportcard.com/report/aptly-dev/aptly
api/api.go
+74 -12
View File
@@ -3,17 +3,18 @@ package api
import (
"fmt"
"log"
"net/http"
"sort"
"strconv"
"strings"
"sync/atomic"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
)
// Lock order acquisition (canonical):
@@ -27,6 +28,23 @@ func apiVersion(c *gin.Context) {
c.JSON(200, gin.H{"Version": aptly.Version})
}
// GET /api/ready
func apiReady(isReady *atomic.Value) func(*gin.Context) {
return func(c *gin.Context) {
if isReady == nil || !isReady.Load().(bool) {
c.JSON(503, gin.H{"Status": "Aptly is unavailable"})
return
}
c.JSON(200, gin.H{"Status": "Aptly is ready"})
}
}
// GET /api/healthy
func apiHealthy(c *gin.Context) {
c.JSON(200, gin.H{"Status": "Aptly is healthy"})
}
type dbRequestKind int
const (
@@ -137,20 +155,29 @@ func maybeRunTaskInBackground(c *gin.Context, name string, resources []string, p
// Run this task in background if configured globally or per-request
background := truthy(c.DefaultQuery("_async", strconv.FormatBool(context.Config().AsyncAPI)))
if background {
log.Println("Executing task asynchronously")
log.Debug().Msg("Executing task asynchronously")
task, conflictErr := runTaskInBackground(name, resources, proc)
if conflictErr != nil {
c.AbortWithError(409, conflictErr)
AbortWithJSONError(c, 409, conflictErr)
return
}
c.JSON(202, task)
} else {
log.Println("Executing task synchronously")
out := context.Progress()
detail := task.Detail{}
retValue, err := proc(out, &detail)
log.Debug().Msg("Executing task synchronously")
task, conflictErr := runTaskInBackground(name, resources, proc)
if conflictErr != nil {
AbortWithJSONError(c, 409, conflictErr)
return
}
// wait for task to finish
context.TaskList().WaitForTaskByID(task.ID)
retValue, _ := context.TaskList().GetTaskReturnValueByID(task.ID)
err, _ := context.TaskList().GetTaskErrorByID(task.ID)
context.TaskList().DeleteTaskByID(task.ID)
if err != nil {
c.AbortWithError(retValue.Code, err)
AbortWithJSONError(c, retValue.Code, err)
return
}
if retValue != nil {
@@ -168,7 +195,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -176,7 +203,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
if queryS != "" {
q, err := query.Parse(c.Request.URL.Query().Get("q"))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -193,7 +220,7 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
return
}
}
@@ -203,11 +230,41 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
return
}
}
// filter packages by version
if c.Request.URL.Query().Get("maximumVersion") == "1" {
list.PrepareIndex()
list.ForEach(func(p *deb.Package) error {
versionQ, err := query.Parse(fmt.Sprintf("Name (%s), $Version (<= %s)", p.Name, p.Version))
if err != nil {
fmt.Println("filter packages by version, query string parse err: ", err)
c.AbortWithError(500, fmt.Errorf("unable to parse %s maximum version query string: %s", p.Name, err))
} else {
tmpList, err := list.Filter([]deb.PackageQuery{versionQ}, false,
nil, 0, []string{})
if err == nil {
if tmpList.Len() > 0 {
tmpList.ForEach(func(tp *deb.Package) error {
list.Remove(tp)
return nil
})
list.Add(p)
}
} else {
fmt.Println("filter packages by version, filter err: ", err)
c.AbortWithError(500, fmt.Errorf("unable to get %s maximum version: %s", p.Name, err))
}
}
return nil
})
}
if c.Request.URL.Query().Get("format") == "details" {
list.ForEach(func(p *deb.Package) error {
result = append(result, p)
@@ -219,3 +276,8 @@ func showPackages(c *gin.Context, reflist *deb.PackageRefList, collectionFactory
c.JSON(200, list.Strings())
}
}
func AbortWithJSONError(c *gin.Context, code int, err error) *gin.Error {
c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8")
return c.AbortWithError(code, err)
}
api/api_test.go
+67 -9
View File
@@ -1,16 +1,20 @@
package api
import (
"bytes"
"encoding/json"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/gin-gonic/gin"
"github.com/smira/flag"
. "gopkg.in/check.v1"
@@ -30,12 +34,13 @@ type ApiSuite struct {
var _ = Suite(&ApiSuite{})
func createTestConfig() *os.File {
file, err := ioutil.TempFile("", "aptly")
file, err := os.CreateTemp("", "aptly")
if err != nil {
return nil
}
jsonString, err := json.Marshal(gin.H{
"architectures": []string{},
"architectures": []string{},
"enableMetricsEndpoint": true,
})
if err != nil {
return nil
@@ -44,9 +49,12 @@ func createTestConfig() *os.File {
return file
}
func (s *ApiSuite) SetUpSuite(c *C) {
func (s *ApiSuite) setupContext() error {
aptly.Version = "testVersion"
file := createTestConfig()
c.Assert(file, NotNil)
if nil == file {
return fmt.Errorf("unable to create the test configuration file")
}
s.configFile = file
flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError)
@@ -57,10 +65,19 @@ func (s *ApiSuite) SetUpSuite(c *C) {
s.flags = flags
context, err := ctx.NewContext(s.flags)
c.Assert(err, IsNil)
if nil != err {
return err
}
s.context = context
s.router = Router(context)
return nil
}
func (s *ApiSuite) SetUpSuite(c *C) {
err := s.setupContext()
c.Assert(err, IsNil)
}
func (s *ApiSuite) TearDownSuite(c *C) {
@@ -85,11 +102,52 @@ func (s *ApiSuite) HTTPRequest(method string, url string, body io.Reader) (*http
return w, nil
}
func (s *ApiSuite) TestGinRunsInReleaseMode(c *C) {
c.Check(gin.Mode(), Equals, gin.ReleaseMode)
}
func (s *ApiSuite) TestGetVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/version", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, ".*Version.*")
c.Check(response.Body.String(), Matches, "{\"Version\":\""+aptly.Version+"\"}")
}
func (s *ApiSuite) TestGetReadiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/ready", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is ready\"}")
}
func (s *ApiSuite) TestGetHealthiness(c *C) {
response, err := s.HTTPRequest("GET", "/api/healthy", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Matches, "{\"Status\":\"Aptly is healthy\"}")
}
func (s *ApiSuite) TestGetMetrics(c *C) {
response, err := s.HTTPRequest("GET", "/api/metrics", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
b := strings.Replace(response.Body.String(), "\n", "", -1)
c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_in_flight gauge.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_requests_total counter.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_request_size_bytes summary.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_response_size_bytes summary.*")
c.Check(b, Matches, ".*# TYPE aptly_api_http_request_duration_seconds summary.*")
c.Check(b, Matches, ".*# TYPE aptly_build_info gauge.*")
c.Check(b, Matches, ".*aptly_build_info.*version=\"testVersion\".*")
}
func (s *ApiSuite) TestRepoCreate(c *C) {
body, err := json.Marshal(gin.H{
"Name": "dummy",
})
c.Assert(err, IsNil)
_, err = s.HTTPRequest("POST", "/api/repos", bytes.NewReader(body))
c.Assert(err, IsNil)
}
func (s *ApiSuite) TestTruthy(c *C) {
api/error.go
+5
View File
@@ -0,0 +1,5 @@
package api
type Error struct {
Error string `json:"error"`
}
api/files.go
+37 -24
View File
@@ -6,8 +6,11 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
"github.com/saracen/walker"
)
func verifyPath(path string) bool {
@@ -24,27 +27,31 @@ func verifyPath(path string) bool {
func verifyDir(c *gin.Context) bool {
if !verifyPath(c.Params.ByName("dir")) {
c.AbortWithError(400, fmt.Errorf("wrong dir"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong dir"))
return false
}
return true
}
// GET /files
// @Summary Get files
// @Description Get list of uploaded files.
// @Tags Files
// @Produce json
// @Success 200 {array} string "List of files"
// @Router /api/files [get]
func apiFilesListDirs(c *gin.Context) {
list := []string{}
listLock := &sync.Mutex{}
err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error {
if path == context.UploadPath() {
return nil
}
if info.IsDir() {
listLock.Lock()
defer listLock.Unlock()
list = append(list, filepath.Base(path))
return filepath.SkipDir
}
@@ -53,7 +60,7 @@ func apiFilesListDirs(c *gin.Context) {
})
if err != nil && !os.IsNotExist(err) {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -66,17 +73,17 @@ func apiFilesUpload(c *gin.Context) {
return
}
path := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
err := os.MkdirAll(path, 0777)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = c.Request.ParseMultipartForm(10 * 1024 * 1024)
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -86,7 +93,7 @@ func apiFilesUpload(c *gin.Context) {
for _, file := range files {
src, err := file.Open()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
defer src.Close()
@@ -94,14 +101,14 @@ func apiFilesUpload(c *gin.Context) {
destPath := filepath.Join(path, filepath.Base(file.Filename))
dst, err := os.Create(destPath)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -109,6 +116,7 @@ func apiFilesUpload(c *gin.Context) {
}
}
apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc()
c.JSON(200, stored)
}
@@ -120,9 +128,10 @@ func apiFilesListFiles(c *gin.Context) {
}
list := []string{}
root := filepath.Join(context.UploadPath(), c.Params.ByName("dir"))
listLock := &sync.Mutex{}
root := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir")))
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
err := filepath.Walk(root, func(path string, _ os.FileInfo, err error) error {
if err != nil {
return err
}
@@ -131,6 +140,8 @@ func apiFilesListFiles(c *gin.Context) {
return nil
}
listLock.Lock()
defer listLock.Unlock()
list = append(list, filepath.Base(path))
return nil
@@ -138,9 +149,9 @@ func apiFilesListFiles(c *gin.Context) {
if err != nil {
if os.IsNotExist(err) {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
} else {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
}
return
}
@@ -154,9 +165,9 @@ func apiFilesDeleteDir(c *gin.Context) {
return
}
err := os.RemoveAll(filepath.Join(context.UploadPath(), c.Params.ByName("dir")))
err := os.RemoveAll(filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))))
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -169,15 +180,17 @@ func apiFilesDeleteFile(c *gin.Context) {
return
}
if !verifyPath(c.Params.ByName("name")) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
dir := utils.SanitizePath(c.Params.ByName("dir"))
name := utils.SanitizePath(c.Params.ByName("name"))
if !verifyPath(name) {
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
err := os.Remove(filepath.Join(context.UploadPath(), c.Params.ByName("dir"), c.Params.ByName("name")))
err := os.Remove(filepath.Join(context.UploadPath(), dir, name))
if err != nil {
if err1, ok := err.(*os.PathError); !ok || !os.IsNotExist(err1.Err) {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
}
api/gpg.go
+16 -12
View File
@@ -2,13 +2,13 @@ package api
import (
"fmt"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
)
@@ -24,9 +24,13 @@ func apiGPGAddKey(c *gin.Context) {
if c.Bind(&b) != nil {
return
}
b.Keyserver = utils.SanitizePath(b.Keyserver)
b.GpgKeyID = utils.SanitizePath(b.GpgKeyID)
b.GpgKeyArmor = utils.SanitizePath(b.GpgKeyArmor)
// b.Keyring can be an absolute path
var err error
args := []string{"--no-default-keyring"}
args := []string{"--no-default-keyring", "--allow-non-selfsigned-uid"}
keyring := "trustedkeys.gpg"
if len(b.Keyring) > 0 {
keyring = b.Keyring
@@ -37,9 +41,9 @@ func apiGPGAddKey(c *gin.Context) {
}
if len(b.GpgKeyArmor) > 0 {
var tempdir string
tempdir, err = ioutil.TempDir(os.TempDir(), "aptly")
tempdir, err = os.MkdirTemp(os.TempDir(), "aptly")
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
defer os.RemoveAll(tempdir)
@@ -47,11 +51,11 @@ func apiGPGAddKey(c *gin.Context) {
keypath := filepath.Join(tempdir, "key")
keyfile, e := os.Create(keypath)
if e != nil {
c.AbortWithError(400, e)
AbortWithJSONError(c, 400, e)
return
}
if _, e = keyfile.WriteString(b.GpgKeyArmor); e != nil {
c.AbortWithError(400, e)
AbortWithJSONError(c, 400, e)
}
args = append(args, "--import", keypath)
@@ -62,10 +66,10 @@ func apiGPGAddKey(c *gin.Context) {
args = append(args, keys...)
}
finder := pgp.GPG1Finder()
finder := pgp.GPGDefaultFinder()
gpg, _, err := finder.FindGPG()
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -74,11 +78,11 @@ func apiGPGAddKey(c *gin.Context) {
// there is no error handling for such as gpg will do this for us
cmd := exec.Command(gpg, args...)
fmt.Printf("running %s %s\n", gpg, strings.Join(args, " "))
cmd.Stdout = os.Stdout
if err = cmd.Run(); err != nil {
c.AbortWithError(400, err)
out, err := cmd.CombinedOutput()
if err != nil {
c.JSON(400, string(out))
return
}
c.JSON(200, gin.H{})
c.JSON(200, string(out))
}
api/graph.go
+4 -4
View File
@@ -43,25 +43,25 @@ func apiGraph(c *gin.Context) {
stdin, err := command.StdinPipe()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
_, err = io.Copy(stdin, buf)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = stdin.Close()
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
output, err = command.Output()
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to execute dot: %s (is graphviz package installed?)", err))
return
}
api/metrics.go
+116
View File
@@ -0,0 +1,116 @@
package api
import (
"fmt"
"runtime"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/deb"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/rs/zerolog/log"
)
var (
apiRequestsInFlightGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_api_http_requests_in_flight",
Help: "Number of concurrent HTTP api requests currently handled.",
},
[]string{"method", "path"},
)
apiRequestsTotalCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_http_requests_total",
Help: "Total number of api requests.",
},
[]string{"code", "method", "path"},
)
apiRequestSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_size_bytes",
Help: "Api HTTP request size in bytes.",
},
[]string{"code", "method", "path"},
)
apiResponseSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_response_size_bytes",
Help: "Api HTTP response size in bytes.",
},
[]string{"code", "method", "path"},
)
apiRequestsDurationSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_duration_seconds",
Help: "Duration of api requests in seconds.",
},
[]string{"code", "method", "path"},
)
apiVersionGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_build_info",
Help: "Metric with a constant '1' value labeled by version and goversion from which aptly was built.",
},
[]string{"version", "goversion"},
)
apiFilesUploadedCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_files_uploaded_total",
Help: "Total number of uploaded files labeled by upload directory.",
},
[]string{"directory"},
)
apiReposPackageCountGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_repos_package_count",
Help: "Current number of published packages labeled by source, distribution and component.",
},
[]string{"source", "distribution", "component"},
)
)
type metricsCollectorRegistrar struct {
hasRegistered bool
}
func (r *metricsCollectorRegistrar) Register(router *gin.Engine) {
if !r.hasRegistered {
apiVersionGauge.WithLabelValues(aptly.Version, runtime.Version()).Set(1)
router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
r.hasRegistered = true
}
}
var MetricsCollectorRegistrar = metricsCollectorRegistrar{hasRegistered: false}
func countPackagesByRepos() {
err := context.NewCollectionFactory().PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
err := context.NewCollectionFactory().PublishedRepoCollection().LoadComplete(repo, context.NewCollectionFactory())
if err != nil {
msg := fmt.Sprintf(
"Error %s found while determining package count for metrics endpoint (prefix:%s / distribution:%s / component:%s\n).",
err, repo.StoragePrefix(), repo.Distribution, repo.Components())
log.Warn().Msg(msg)
return err
}
components := repo.Components()
for _, c := range components {
count := float64(len(repo.RefList(c).Refs))
apiReposPackageCountGauge.WithLabelValues(fmt.Sprintf("%s", (repo.SourceNames())), repo.Distribution, c).Set(count)
}
return nil
})
if err != nil {
msg := fmt.Sprintf("Error %s found while listing published repos for metrics endpoint", err)
log.Warn().Msg(msg)
}
}
api/middleware.go
+58 -45
View File
@@ -9,59 +9,36 @@ import (
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
var (
apiRequestsInFlightGauge = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "aptly_api_http_requests_in_flight",
Help: "Number of concurrent HTTP api requests currently handled.",
},
[]string{"method", "path"},
)
apiRequestsTotalCounter = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "aptly_api_http_requests_total",
Help: "Total number of api requests.",
},
[]string{"code", "method", "path"},
)
apiRequestSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_size_bytes",
Help: "Api HTTP request size in bytes.",
},
[]string{"code", "method", "path"},
)
apiResponseSizeSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_response_size_bytes",
Help: "Api HTTP response size in bytes.",
},
[]string{"code", "method", "path"},
)
apiRequestsDurationSummary = promauto.NewSummaryVec(
prometheus.SummaryOpts{
Name: "aptly_api_http_request_duration_seconds",
Help: "Duration of api requests in seconds.",
},
[]string{"code", "method", "path"},
)
"github.com/rs/zerolog/log"
)
// Only use base path as label value (e.g.: /api/repos) because of time series cardinality
// See https://prometheus.io/docs/practices/naming/#labels
func getBasePath(c *gin.Context) string {
return fmt.Sprintf("%s%s", getURLSegment(c.Request.URL.Path, 0), getURLSegment(c.Request.URL.Path, 1))
segment0, err := getURLSegment(c.Request.URL.Path, 0)
if err != nil {
return "/"
}
segment1, err := getURLSegment(c.Request.URL.Path, 1)
if err != nil {
return *segment0
}
return *segment0 + *segment1
}
func getURLSegment(url string, idx int) string {
var urlSegments = strings.Split(url, "/")
func getURLSegment(url string, idx int) (*string, error) {
urlSegments := strings.Split(url, "/")
// Remove segment at index 0 because it's an empty string
var segmentAtIndex = urlSegments[1:cap(urlSegments)][idx]
return fmt.Sprintf("/%s", segmentAtIndex)
urlSegments = urlSegments[1:cap(urlSegments)]
if len(urlSegments) <= idx {
return nil, fmt.Errorf("index %d out of range, only has %d url segments", idx, len(urlSegments))
}
segmentAtIndex := urlSegments[idx]
s := fmt.Sprintf("/%s", segmentAtIndex)
return &s, nil
}
func instrumentHandlerInFlight(g *prometheus.GaugeVec, pathFunc func(*gin.Context) string) func(*gin.Context) {
@@ -101,3 +78,39 @@ func instrumentHandlerDuration(obs prometheus.ObserverVec, pathFunc func(*gin.Co
obs.WithLabelValues(strconv.Itoa(c.Writer.Status()), c.Request.Method, pathFunc(c)).Observe(time.Since(now).Seconds())
}
}
// JSONLogger is a gin middleware that takes an instance of Logger and uses it for writing access
// logs that include error messages if there are any.
func JSONLogger() gin.HandlerFunc {
return func(c *gin.Context) {
// Start timer
start := time.Now()
path := c.Request.URL.Path
raw := c.Request.URL.RawQuery
// Process request
c.Next()
ts := time.Now()
if raw != "" {
path = path + "?" + raw
}
errorMessage := strings.TrimSuffix(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
l := log.With().Str("remote", c.ClientIP()).Logger().
With().Str("method", c.Request.Method).Logger().
With().Str("path", path).Logger().
With().Str("protocol", c.Request.Proto).Logger().
With().Str("code", fmt.Sprint(c.Writer.Status())).Logger().
With().Str("latency", ts.Sub(start).String()).Logger().
With().Str("agent", c.Request.UserAgent()).Logger()
if c.Writer.Status() >= 400 && c.Writer.Status() < 500 {
l.Warn().Msg(errorMessage)
} else if c.Writer.Status() >= 500 {
l.Error().Msg(errorMessage)
} else {
l.Info().Msg(errorMessage)
}
}
}
api/middleware_test.go
+255
View File
@@ -0,0 +1,255 @@
package api
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"sync/atomic"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
. "gopkg.in/check.v1"
)
type MiddlewareSuite struct {
router http.Handler
context *gin.Context
logReader *os.File
logWriter *os.File
}
var _ = Suite(&MiddlewareSuite{})
func (s *MiddlewareSuite) SetUpTest(c *C) {
r, w, err := os.Pipe()
c.Assert(err, IsNil)
utils.SetupJSONLogger("debug", w)
mw := JSONLogger()
router := gin.New()
router.UseRawPath = true
router.Use(mw)
router.Use(gin.Recovery(), gin.ErrorLogger())
root := router.Group("/api")
isReady := &atomic.Value{}
isReady.Store(false)
root.GET("/ready", apiReady(isReady))
root.GET("/healthy", apiHealthy)
s.router = router
s.logReader = r
s.logWriter = w
}
func (s *MiddlewareSuite) TearDownTest(c *C) {
s.router = nil
s.context = nil
s.logReader = nil
s.logWriter = nil
}
func (s *MiddlewareSuite) HTTPRequest(method string, url string, body io.Reader) {
recorder := httptest.NewRecorder()
s.context, _ = gin.CreateTestContext(recorder)
req, _ := http.NewRequestWithContext(s.context, method, url, body)
s.context.Request = req
req.Header.Add("Content-Type", "application/json")
s.router.ServeHTTP(httptest.NewRecorder(), req)
}
func (s *MiddlewareSuite) TestJSONMiddleware4xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "warn")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["method"]; ok {
c.Check(val, Equals, "GET")
} else {
c.Errorf("Log message didn't have a 'method' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["path"]; ok {
c.Check(val, Equals, "/")
} else {
c.Errorf("Log message didn't have a 'path' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["protocol"]; ok {
c.Check(val, Equals, "HTTP/1.1")
} else {
c.Errorf("Log message didn't have a 'protocol' key, obtained %s", capturedOutput)
}
if val, ok := jsonMap["code"]; ok {
c.Check(val, Equals, "404")
} else {
c.Errorf("Log message didn't have a 'code' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["remote"]; !ok {
c.Errorf("Log message didn't have a 'remote' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["latency"]; !ok {
c.Errorf("Log message didn't have a 'latency' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["agent"]; !ok {
c.Errorf("Log message didn't have a 'agent' key, obtained %s", capturedOutput)
}
if _, ok := jsonMap["time"]; !ok {
c.Errorf("Log message didn't have a 'time' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddleware2xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddleware5xx(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/ready", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "error")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestJSONMiddlewareRaw(c *C) {
outC := make(chan string)
go func() {
var buf bytes.Buffer
io.Copy(&buf, s.logReader)
fmt.Println(buf.String())
outC <- buf.String()
}()
s.HTTPRequest(http.MethodGet, "/api/healthy?test=raw", nil)
s.logWriter.Close()
capturedOutput := <-outC
var jsonMap map[string]interface{}
json.Unmarshal([]byte(capturedOutput), &jsonMap)
fmt.Println(capturedOutput)
if val, ok := jsonMap["level"]; ok {
c.Check(val, Equals, "info")
} else {
c.Errorf("Log message didn't have a 'level' key, obtained %s", capturedOutput)
}
}
func (s *MiddlewareSuite) TestGetBasePath(c *C) {
s.HTTPRequest(http.MethodGet, "", nil)
path := getBasePath(s.context)
c.Check(path, Equals, "/")
s.HTTPRequest(http.MethodGet, "/", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/")
s.HTTPRequest(http.MethodGet, "/api", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/api")
s.HTTPRequest(http.MethodGet, "/api/repos/testRepo", nil)
path = getBasePath(s.context)
c.Check(path, Equals, "/api/repos")
}
func (s *MiddlewareSuite) TestGetURLSegment(c *C) {
url := "/"
segment, err := getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/")
_, err = getURLSegment(url, 1)
if err == nil {
c.Error("Invalid return value")
}
url = "/api"
segment, err = getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/api")
_, err = getURLSegment(url, 1)
if err == nil {
c.Error("Invalid return value")
}
url = "/api/repos/testRepo"
segment, err = getURLSegment(url, 0)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/api")
segment, err = getURLSegment(url, 1)
if err != nil {
c.Error(err)
}
c.Check(*segment, Equals, "/repos")
}
api/mirror.go
+202 -99
View File
@@ -2,8 +2,8 @@ package api
import (
"fmt"
"log"
"net/http"
"os"
"sort"
"strings"
"sync"
@@ -14,19 +14,16 @@ import (
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
"github.com/rs/zerolog/log"
)
func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error) {
if ignoreSignatures {
return nil, nil
}
func getVerifier(keyRings []string) (pgp.Verifier, error) {
verifier := context.GetVerifier()
for _, keyRing := range keyRings {
verifier.AddKeyring(keyRing)
}
err := verifier.InitKeyring()
err := verifier.InitKeyring(false)
if err != nil {
return nil, err
}
@@ -34,7 +31,13 @@ func getVerifier(ignoreSignatures bool, keyRings []string) (pgp.Verifier, error)
return verifier, nil
}
// GET /api/mirrors
// @Summary Get mirrors
// @Description **Show list of currently available mirrors**
// @Description Each mirror is returned as in “show” API.
// @Tags Mirrors
// @Produce json
// @Success 200 {array} deb.RemoteRepo
// @Router /api/mirrors [get]
func apiMirrorsList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -48,24 +51,49 @@ func apiMirrorsList(c *gin.Context) {
c.JSON(200, result)
}
// POST /api/mirrors
type mirrorCreateParams struct {
// Name of mirror to be created
Name string `binding:"required" json:"Name" example:"mirror2"`
// Url of the archive to mirror
ArchiveURL string `binding:"required" json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Distribution name to mirror
Distribution string ` json:"Distribution" example:"'buster', for flat repositories use './'"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Gpg keyring(s) for verifying Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to mirror installer files
DownloadInstaller bool ` json:"DownloadInstaller"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to skip if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip the verification of architectures
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
}
// @Summary Create mirror
// @Description **Create a mirror**
// @Tags Mirrors
// @Consume json
// @Param request body mirrorCreateParams true "Parameters"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 400 {object} Error "Bad Request"
// @Router /api/mirrors [post]
func apiMirrorsCreate(c *gin.Context) {
var err error
var b struct {
Name string `binding:"required"`
ArchiveURL string `binding:"required"`
Distribution string
Filter string
Components []string
Architectures []string
Keyrings []string
DownloadSources bool
DownloadUdebs bool
DownloadInstaller bool
FilterWithDeps bool
SkipComponentCheck bool
IgnoreSignatures bool
}
var b mirrorCreateParams
b.DownloadSources = context.Config().DownloadSourcePackages
b.IgnoreSignatures = context.Config().GpgDisableVerify
@@ -81,7 +109,7 @@ func apiMirrorsCreate(c *gin.Context) {
if strings.HasPrefix(b.ArchiveURL, "ppa:") {
b.ArchiveURL, b.Distribution, b.Components, err = deb.ParsePPA(b.ArchiveURL, context.Config())
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
}
@@ -89,7 +117,7 @@ func apiMirrorsCreate(c *gin.Context) {
if b.Filter != "" {
_, err = query.Parse(b.Filter)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
return
}
}
@@ -98,39 +126,50 @@ func apiMirrorsCreate(c *gin.Context) {
b.DownloadSources, b.DownloadUdebs, b.DownloadInstaller)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to create mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to create mirror: %s", err))
return
}
repo.Filter = b.Filter
repo.FilterWithDeps = b.FilterWithDeps
repo.SkipComponentCheck = b.SkipComponentCheck
repo.SkipArchitectureCheck = b.SkipArchitectureCheck
repo.DownloadSources = b.DownloadSources
repo.DownloadUdebs = b.DownloadUdebs
verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
verifier, err := getVerifier(b.Keyrings)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
downloader := context.NewDownloader(nil)
err = repo.Fetch(downloader, verifier)
err = repo.Fetch(downloader, verifier, b.IgnoreSignatures)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to fetch mirror: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to fetch mirror: %s", err))
return
}
err = collection.Add(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to add mirror: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to add mirror: %s", err))
return
}
c.JSON(201, repo)
}
// DELETE /api/mirrors/:name
// @Summary Delete Mirror
// @Description **Delete a mirror**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Param force query int true "force: 1 to enable"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue
// @Failure 404 {object} Error "Mirror not found"
// @Failure 403 {object} Error "Unable to delete mirror with snapshots"
// @Failure 500 {object} Error "Unable to delete"
// @Router /api/mirrors/{name} [delete]
func apiMirrorsDrop(c *gin.Context) {
name := c.Params.ByName("name")
force := c.Request.URL.Query().Get("force") == "1"
@@ -141,13 +180,13 @@ func apiMirrorsDrop(c *gin.Context) {
repo, err := mirrorCollection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to drop: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to drop: %s", err))
return
}
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete mirror %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := repo.CheckLock()
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err)
@@ -157,7 +196,7 @@ func apiMirrorsDrop(c *gin.Context) {
snapshots := snapshotCollection.ByRemoteRepoSource(repo)
if len(snapshots) > 0 {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override")
}
}
@@ -169,7 +208,15 @@ func apiMirrorsDrop(c *gin.Context) {
})
}
// GET /api/mirrors/:name
// @Summary Show Mirror
// @Description **Get mirror information by name**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Produce json
// @Success 200 {object} deb.RemoteRepo
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [get]
func apiMirrorsShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -177,19 +224,30 @@ func apiMirrorsShow(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
}
c.JSON(200, repo)
}
// GET /api/mirrors/:name/packages
// @Summary List Mirror Packages
// @Description **Get a list of packages from a mirror**
// @Tags Mirrors
// @Param name path string true "mirror name"
// @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information"
// @Produce json
// @Success 200 {array} deb.Package "List of Packages"
// @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name}/packages [get]
func apiMirrorsPackages(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
@@ -197,17 +255,17 @@ func apiMirrorsPackages(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show: %s", err))
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to show: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err))
}
if repo.LastDownloadDate.IsZero() {
c.AbortWithError(404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
AbortWithJSONError(c, 404, fmt.Errorf("unable to show package list, mirror hasn't been downloaded yet"))
return
}
@@ -216,7 +274,7 @@ func apiMirrorsPackages(c *gin.Context) {
list, err := deb.NewPackageListFromRefList(reflist, collectionFactory.PackageCollection(), nil)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -224,7 +282,7 @@ func apiMirrorsPackages(c *gin.Context) {
if queryS != "" {
q, err := query.Parse(c.Request.URL.Query().Get("q"))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -241,7 +299,7 @@ func apiMirrorsPackages(c *gin.Context) {
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
c.AbortWithError(400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to determine list of architectures, please specify explicitly"))
return
}
}
@@ -251,7 +309,7 @@ func apiMirrorsPackages(c *gin.Context) {
list, err = list.Filter([]deb.PackageQuery{q}, withDeps,
nil, context.DependencyOptions(), architecturesList)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to search: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to search: %s", err))
}
}
@@ -267,36 +325,65 @@ func apiMirrorsPackages(c *gin.Context) {
}
}
// PUT /api/mirrors/:name
type mirrorUpdateParams struct {
// Change mirror name to `Name`
Name string ` json:"Name" example:"mirror1"`
// Url of the archive to mirror
ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"`
// Package query that is applied to mirror packages
Filter string ` json:"Filter" example:"xserver-xorg"`
// Limit mirror to those architectures, if not specified aptly would fetch all architectures
Architectures []string ` json:"Architectures" example:"amd64"`
// Components to mirror, if not specified aptly would fetch all components
Components []string ` json:"Components" example:"main"`
// Gpg keyring(s) for verifing Release file
Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"`
// Set "true" to include dependencies of matching packages when filtering
FilterWithDeps bool ` json:"FilterWithDeps"`
// Set "true" to mirror source packages
DownloadSources bool ` json:"DownloadSources"`
// Set "true" to mirror udeb files
DownloadUdebs bool ` json:"DownloadUdebs"`
// Set "true" to skip checking if the given components are in the Release file
SkipComponentCheck bool ` json:"SkipComponentCheck"`
// Set "true" to skip checking if the given architectures are in the Release file
SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"`
// Set "true" to ignore checksum errors
IgnoreChecksums bool ` json:"IgnoreChecksums"`
// Set "true" to skip the verification of Release file signatures
IgnoreSignatures bool ` json:"IgnoreSignatures"`
// Set "true" to force a mirror update even if another process is already updating the mirror (use with caution!)
ForceUpdate bool ` json:"ForceUpdate"`
// Set "true" to skip downloading already downloaded packages
SkipExistingPackages bool ` json:"SkipExistingPackages"`
}
// @Summary Update Mirror
// @Description **Update Mirror and download packages**
// @Tags Mirrors
// @Param name path string true "mirror name to update"
// @Consume json
// @Param request body mirrorUpdateParams true "Parameters"
// @Produce json
// @Success 200 {object} task.ProcessReturnValue "Mirror was updated successfully"
// @Success 202 {object} task.Task "Mirror is being updated"
// @Failure 400 {object} Error "Unable to determine list of architectures"
// @Failure 404 {object} Error "Mirror not found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/mirrors/{name} [put]
func apiMirrorsUpdate(c *gin.Context) {
var (
err error
remote *deb.RemoteRepo
b mirrorUpdateParams
)
var b struct {
Name string
ArchiveURL string
Filter string
Architectures []string
Components []string
Keyrings []string
FilterWithDeps bool
DownloadSources bool
DownloadUdebs bool
SkipComponentCheck bool
IgnoreChecksums bool
IgnoreSignatures bool
ForceUpdate bool
SkipExistingPackages bool
}
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.RemoteRepoCollection()
remote, err = collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -304,12 +391,14 @@ func apiMirrorsUpdate(c *gin.Context) {
b.DownloadUdebs = remote.DownloadUdebs
b.DownloadSources = remote.DownloadSources
b.SkipComponentCheck = remote.SkipComponentCheck
b.SkipArchitectureCheck = remote.SkipArchitectureCheck
b.FilterWithDeps = remote.FilterWithDeps
b.Filter = remote.Filter
b.Architectures = remote.Architectures
b.Components = remote.Components
b.IgnoreSignatures = context.Config().GpgDisableVerify
log.Printf("%s: Starting mirror update\n", b.Name)
log.Info().Msgf("%s: Starting mirror update", b.Name)
if c.Bind(&b) != nil {
return
@@ -318,14 +407,14 @@ func apiMirrorsUpdate(c *gin.Context) {
if b.Name != remote.Name {
_, err = collection.ByName(b.Name)
if err == nil {
c.AbortWithError(409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: mirror %s already exists", b.Name))
return
}
}
if b.DownloadUdebs != remote.DownloadUdebs {
if remote.IsFlat() && b.DownloadUdebs {
c.AbortWithError(400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs"))
return
}
}
@@ -338,14 +427,15 @@ func apiMirrorsUpdate(c *gin.Context) {
remote.DownloadUdebs = b.DownloadUdebs
remote.DownloadSources = b.DownloadSources
remote.SkipComponentCheck = b.SkipComponentCheck
remote.SkipArchitectureCheck = b.SkipArchitectureCheck
remote.FilterWithDeps = b.FilterWithDeps
remote.Filter = b.Filter
remote.Architectures = b.Architectures
remote.Components = b.Components
verifier, err := getVerifier(b.IgnoreSignatures, b.Keyrings)
verifier, err := getVerifier(b.Keyrings)
if err != nil {
c.AbortWithError(400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err))
return
}
@@ -353,7 +443,7 @@ func apiMirrorsUpdate(c *gin.Context) {
maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
downloader := context.NewDownloader(out)
err := remote.Fetch(downloader, verifier)
err := remote.Fetch(downloader, verifier, b.IgnoreSignatures)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -365,7 +455,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.SkipComponentCheck)
err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
@@ -458,7 +548,7 @@ func apiMirrorsUpdate(c *gin.Context) {
}
}()
log.Printf("%s: Spawning background processes...\n", b.Name)
log.Info().Msgf("%s: Spawning background processes...", b.Name)
var wg sync.WaitGroup
for i := 0; i < context.Config().DownloadConcurrency; i++ {
wg.Add(1)
@@ -476,7 +566,16 @@ func apiMirrorsUpdate(c *gin.Context) {
var e error
// provision download location
task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
} else {
var file *os.File
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
file.Close()
}
}
if e != nil {
pushError(e)
continue
@@ -494,6 +593,20 @@ func apiMirrorsUpdate(c *gin.Context) {
continue
}
// and import it back to the pool
task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
//return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
pushError(err)
continue
}
// update "attached" files if any
for _, additionalAtask := range task.Additional {
additionalAtask.File.PoolPath = task.File.PoolPath
additionalAtask.File.Checksums = task.File.Checksums
}
task.Done = true
taskFinished <- task
case <-context.Done():
@@ -505,32 +618,22 @@ func apiMirrorsUpdate(c *gin.Context) {
}
// Wait for all download goroutines to finish
log.Printf("%s: Waiting for background processes to finish...\n", b.Name)
log.Info().Msgf("%s: Waiting for background processes to finish...", b.Name)
wg.Wait()
log.Printf("%s: Background processes finished\n", b.Name)
log.Info().Msgf("%s: Background processes finished", b.Name)
close(taskFinished)
for idx := range queue {
defer func() {
for _, task := range queue {
if task.TempDownPath == "" {
continue
}
atask := &queue[idx]
if !atask.Done {
// download not finished yet
continue
if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
}
}
// and import it back to the pool
atask.File.PoolPath, err = context.PackagePool().Import(atask.TempDownPath, atask.File.Filename, &atask.File.Checksums, true, collectionFactory.ChecksumCollection(nil))
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err)
}
// update "attached" files if any
for _, additionalAtask := range atask.Additional {
additionalAtask.File.PoolPath = atask.File.PoolPath
additionalAtask.File.Checksums = atask.File.Checksums
}
}
}()
select {
case <-context.Done():
@@ -539,18 +642,18 @@ func apiMirrorsUpdate(c *gin.Context) {
}
if len(errors) > 0 {
log.Printf("%s: Unable to update because of previous errors\n", b.Name)
log.Info().Msgf("%s: Unable to update because of previous errors", b.Name)
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: download errors:\n %s", strings.Join(errors, "\n "))
}
log.Printf("%s: Finalizing download\n", b.Name)
log.Info().Msgf("%s: Finalizing download...", b.Name)
remote.FinalizeDownload(collectionFactory, out)
err = collectionFactory.RemoteRepoCollection().Update(remote)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
}
log.Printf("%s: Mirror updated successfully!\n", b.Name)
log.Info().Msgf("%s: Mirror updated successfully", b.Name)
return &task.ProcessReturnValue{Code: http.StatusNoContent, Value: nil}, nil
})
}
api/packages.go
+16 -1
View File
@@ -9,9 +9,24 @@ func apiPackagesShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
p, err := collectionFactory.PackageCollection().ByKey([]byte(c.Params.ByName("key")))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
c.JSON(200, p)
}
// @Summary Get packages
// @Description Get list of packages.
// @Tags Packages
// @Consume json
// @Produce json
// @Param q query string false "search query"
// @Param format query string false "format: `details` for more detailed information"
// @Success 200 {array} string "List of packages"
// @Router /api/packages [get]
func apiPackages(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PackageCollection()
showPackages(c, collection.AllPackageRefs(), collectionFactory)
}
api/packages_test.go
+18
View File
@@ -0,0 +1,18 @@
package api
import (
. "gopkg.in/check.v1"
)
type PackagesSuite struct {
ApiSuite
}
var _ = Suite(&PackagesSuite{})
func (s *PackagesSuite) TestPackagesGetMaximumVersion(c *C) {
response, err := s.HTTPRequest("GET", "/api/repos/dummy/packages?maximumVersion=1", nil)
c.Assert(err, IsNil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "[]")
}
api/publish.go
+48 -38
View File
@@ -16,7 +16,6 @@ import (
// SigningOptions is a shared between publish API GPG options structure
type SigningOptions struct {
Skip bool
Batch bool
GpgKey string
Keyring string
SecretKeyring string
@@ -33,7 +32,9 @@ func getSigner(options *SigningOptions) (pgp.Signer, error) {
signer.SetKey(options.GpgKey)
signer.SetKeyRing(options.Keyring, options.SecretKeyring)
signer.SetPassphrase(options.Passphrase, options.PassphraseFile)
signer.SetBatch(options.Batch)
// If Batch is false, GPG will ask for passphrase on stdin, which would block the api process
signer.SetBatch(true)
err := signer.Init()
if err != nil {
@@ -43,16 +44,23 @@ func getSigner(options *SigningOptions) (pgp.Signer, error) {
return signer, nil
}
// Replace '_' with '/' and double '__' with single '_'
func parseEscapedPath(path string) string {
// Replace '_' with '/' and double '__' with single '_', SanitizePath
func slashEscape(path string) string {
result := strings.Replace(strings.Replace(path, "_", "/", -1), "//", "_", -1)
result = utils.SanitizePath(result)
if result == "" {
result = "."
}
return result
}
// GET /publish
// @Summary Get publish points
// @Description Get list of available publish points. Each publish point is returned as in “show” API.
// @Tags Publish
// @Produce json
// @Success 200 {array} deb.PublishedRepo
// @Failure 500 {object} Error "Internal Error"
// @Router /api/publish [get]
func apiPublishList(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.PublishedRepoCollection()
@@ -60,7 +68,7 @@ func apiPublishList(c *gin.Context) {
result := make([]*deb.PublishedRepo, 0, collection.Len())
err := collection.ForEach(func(repo *deb.PublishedRepo) error {
err := collection.LoadComplete(repo, collectionFactory)
err := collection.LoadShallow(repo, collectionFactory)
if err != nil {
return err
}
@@ -71,7 +79,7 @@ func apiPublishList(c *gin.Context) {
})
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -80,7 +88,7 @@ func apiPublishList(c *gin.Context) {
// POST /publish/:prefix
func apiPublishRepoOrSnapshot(c *gin.Context) {
param := parseEscapedPath(c.Params.ByName("prefix"))
param := slashEscape(c.Params.ByName("prefix"))
storage, prefix := deb.ParsePrefix(param)
var b struct {
@@ -100,20 +108,23 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
Architectures []string
Signing SigningOptions
AcquireByHash *bool
MultiDist bool
}
if c.Bind(&b) != nil {
return
}
b.Distribution = utils.SanitizePath(b.Distribution)
signer, err := getSigner(&b.Signing)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err))
return
}
if len(b.Sources) == 0 {
c.AbortWithError(400, fmt.Errorf("unable to publish: soures are empty"))
AbortWithJSONError(c, 400, fmt.Errorf("unable to publish: soures are empty"))
return
}
@@ -123,7 +134,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
var resources []string
collectionFactory := context.NewCollectionFactory()
if b.SourceKind == "snapshot" {
if b.SourceKind == deb.SourceSnapshot {
var snapshot *deb.Snapshot
snapshotCollection := collectionFactory.SnapshotCollection()
@@ -134,14 +145,14 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
snapshot, err = snapshotCollection.ByName(source.Name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err))
return
}
resources = append(resources, string(snapshot.ResourceKey()))
err = snapshotCollection.LoadComplete(snapshot)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
return
}
@@ -158,26 +169,26 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
localRepo, err = localCollection.ByName(source.Name)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to publish: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to publish: %s", err))
return
}
resources = append(resources, string(localRepo.Key()))
err = localCollection.LoadComplete(localRepo)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
}
sources = append(sources, localRepo)
}
} else {
c.AbortWithError(400, fmt.Errorf("unknown SourceKind"))
AbortWithJSONError(c, 400, fmt.Errorf("unknown SourceKind"))
return
}
published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory)
published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, b.MultiDist)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to publish: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to publish: %s", err))
return
}
@@ -211,7 +222,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
}
published.SkipBz2 = context.Config().SkipBz2Publishing
if b.SkipContents != nil {
if b.SkipBz2 != nil {
published.SkipBz2 = *b.SkipBz2
}
@@ -241,9 +252,9 @@ func apiPublishRepoOrSnapshot(c *gin.Context) {
// PUT /publish/:prefix/:distribution
func apiPublishUpdateSwitch(c *gin.Context) {
param := parseEscapedPath(c.Params.ByName("prefix"))
param := slashEscape(c.Params.ByName("prefix"))
storage, prefix := deb.ParsePrefix(param)
distribution := c.Params.ByName("distribution")
distribution := utils.SanitizePath(c.Params.ByName("distribution"))
var b struct {
ForceOverwrite bool
@@ -256,6 +267,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
Name string `binding:"required"`
}
AcquireByHash *bool
MultiDist *bool
}
if c.Bind(&b) != nil {
@@ -264,7 +276,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
signer, err := getSigner(&b.Signing)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to initialize GPG signer: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to initialize GPG signer: %s", err))
return
}
@@ -273,12 +285,12 @@ func apiPublishUpdateSwitch(c *gin.Context) {
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
c.AbortWithError(404, fmt.Errorf("unable to update: %s", err))
AbortWithJSONError(c, 404, fmt.Errorf("unable to update: %s", err))
return
}
err = collection.LoadComplete(published, collectionFactory)
if err != nil {
c.AbortWithError(500, fmt.Errorf("unable to update: %s", err))
AbortWithJSONError(c, 500, fmt.Errorf("unable to update: %s", err))
return
}
@@ -288,7 +300,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
if published.SourceKind == deb.SourceLocalRepo {
if len(b.Snapshots) > 0 {
c.AbortWithError(400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
AbortWithJSONError(c, 400, fmt.Errorf("snapshots shouldn't be given when updating local repo"))
return
}
updatedComponents = published.Components()
@@ -296,23 +308,17 @@ func apiPublishUpdateSwitch(c *gin.Context) {
published.UpdateLocalRepo(component)
}
} else if published.SourceKind == "snapshot" {
publishedComponents := published.Components()
for _, snapshotInfo := range b.Snapshots {
if !utils.StrSliceHasItem(publishedComponents, snapshotInfo.Component) {
c.AbortWithError(404, fmt.Errorf("component %s is not in published repository", snapshotInfo.Component))
return
}
snapshotCollection := collectionFactory.SnapshotCollection()
snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name)
if err2 != nil {
c.AbortWithError(404, err2)
AbortWithJSONError(c, 404, err2)
return
}
err2 = snapshotCollection.LoadComplete(snapshot)
if err2 != nil {
c.AbortWithError(500, err2)
AbortWithJSONError(c, 500, err2)
return
}
@@ -321,7 +327,7 @@ func apiPublishUpdateSwitch(c *gin.Context) {
updatedSnapshots = append(updatedSnapshots, snapshot.Name)
}
} else {
c.AbortWithError(500, fmt.Errorf("unknown published repository type"))
AbortWithJSONError(c, 500, fmt.Errorf("unknown published repository type"))
return
}
@@ -337,9 +343,13 @@ func apiPublishUpdateSwitch(c *gin.Context) {
published.AcquireByHash = *b.AcquireByHash
}
if b.MultiDist != nil {
published.MultiDist = *b.MultiDist
}
resources = append(resources, string(published.Key()))
taskName := fmt.Sprintf("Update published %s (%s): %s", published.SourceKind, strings.Join(updatedComponents, " "), strings.Join(updatedSnapshots, ", "))
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err)
@@ -367,7 +377,7 @@ func apiPublishDrop(c *gin.Context) {
force := c.Request.URL.Query().Get("force") == "1"
skipCleanup := c.Request.URL.Query().Get("SkipCleanup") == "1"
param := parseEscapedPath(c.Params.ByName("prefix"))
param := slashEscape(c.Params.ByName("prefix"))
storage, prefix := deb.ParsePrefix(param)
distribution := c.Params.ByName("distribution")
@@ -376,14 +386,14 @@ func apiPublishDrop(c *gin.Context) {
published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("unable to drop: %s", err))
AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to drop: %s", err))
return
}
resources := []string{string(published.Key())}
taskName := fmt.Sprintf("Delete published %s (%s)", prefix, distribution)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := collection.Remove(context, storage, prefix, distribution,
collectionFactory, out, force, skipCleanup)
if err != nil {
api/repos.go
+243 -27
View File
@@ -5,6 +5,7 @@ import (
"net/http"
"os"
"path/filepath"
"sort"
"strings"
"text/template"
@@ -17,7 +18,44 @@ import (
"github.com/gin-gonic/gin"
)
// GET /api/repos
// GET /repos
func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc {
return func(c *gin.Context) {
c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8")
c.Writer.Flush()
c.Writer.WriteString("<pre>\n")
if len(localRepos) == 0 {
c.Writer.WriteString("<a href=\"-/\">default</a>\n")
}
for publishPrefix := range localRepos {
c.Writer.WriteString(fmt.Sprintf("<a href=\"%[1]s/\">%[1]s</a>\n", publishPrefix))
}
c.Writer.WriteString("</pre>")
c.Writer.Flush()
}
}
// GET /repos/:storage/*pkgPath
func reposServeInAPIMode(c *gin.Context) {
pkgpath := c.Param("pkgPath")
storage := c.Param("storage")
if storage == "-" {
storage = ""
} else {
storage = "filesystem:" + storage
}
publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath()
c.FileFromFS(pkgpath, http.Dir(publicPath))
}
// @Summary Get repos
// @Description Get list of available repos. Each repo is returned as in “show” API.
// @Tags Repos
// @Produce json
// @Success 200 {array} deb.LocalRepo
// @Router /api/repos [get]
func apiReposList(c *gin.Context) {
result := []*deb.LocalRepo{}
@@ -31,7 +69,18 @@ func apiReposList(c *gin.Context) {
c.JSON(200, result)
}
// POST /api/repos
// @Summary Create repository
// @Description Create a local repository.
// @Tags Repos
// @Produce json
// @Consume json
// @Param Name query string false "Name of repository to be created."
// @Param Comment query string false "Text describing local repository, for the user"
// @Param DefaultDistribution query string false "Default distribution when publishing from this local repo"
// @Param DefaultComponent query string false "Default component when publishing from this local repo"
// @Success 201 {object} deb.LocalRepo
// @Failure 400 {object} Error "Repository already exists"
// @Router /api/repos [post]
func apiReposCreate(c *gin.Context) {
var b struct {
Name string `binding:"required"`
@@ -52,7 +101,7 @@ func apiReposCreate(c *gin.Context) {
collection := collectionFactory.LocalRepoCollection()
err := collection.Add(repo)
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -77,7 +126,7 @@ func apiReposEdit(c *gin.Context) {
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -85,7 +134,7 @@ func apiReposEdit(c *gin.Context) {
_, err := collection.ByName(*b.Name)
if err == nil {
// already exists
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
repo.Name = *b.Name
@@ -102,7 +151,7 @@ func apiReposEdit(c *gin.Context) {
err = collection.Update(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -110,13 +159,21 @@ func apiReposEdit(c *gin.Context) {
}
// GET /api/repos/:name
// @Summary Get repository info by name
// @Description Returns basic information about local repository.
// @Tags Repos
// @Produce json
// @Param name path string true "Repository name"
// @Success 200 {object} deb.LocalRepo
// @Failure 404 {object} Error "Repository not found"
// @Router /api/repos/{name} [get]
func apiReposShow(c *gin.Context) {
collectionFactory := context.NewCollectionFactory()
collection := collectionFactory.LocalRepoCollection()
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -135,13 +192,13 @@ func apiReposDrop(c *gin.Context) {
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(repo.Key())}
taskName := fmt.Sprintf("Delete repo %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
published := publishedCollection.ByLocalRepo(repo)
if len(published) > 0 {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published")
@@ -165,13 +222,13 @@ func apiReposPackagesShow(c *gin.Context) {
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -193,18 +250,18 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li
repo, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
resources := []string{string(repo.Key())}
maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
out.Printf("Loading packages...\n")
list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil)
if err != nil {
@@ -262,7 +319,22 @@ func apiReposPackageFromFile(c *gin.Context) {
apiReposPackageFromDir(c)
}
// POST /repos/:name/file/:dir
// @Summary Add packages from uploaded file/directory
// @Description Import packages from files (uploaded using File Upload API) to the local repository. If directory specified, aptly would discover package files automatically.
// @Description Adding same package to local repository is not an error.
// @Description By default aptly would try to remove every successfully processed file and directory `dir` (if it becomes empty after import).
// @Tags Repos
// @Param name path string true "Repository name"
// @Param dir path string true "Directory to add"
// @Consume json
// @Param noRemove query string false "when value is set to 1, don’t remove any files"
// @Param forceReplace query string false "when value is set to 1, remove packages conflicting with package being added (in local repository)"
// @Produce json
// @Success 200 {string} string "OK"
// @Failure 400 {object} Error "wrong file"
// @Failure 404 {object} Error "Repository not found"
// @Failure 500 {object} Error "Error adding files"
// @Router /api/repos/{name}/{dir} [post]
func apiReposPackageFromDir(c *gin.Context) {
forceReplace := c.Request.URL.Query().Get("forceReplace") == "1"
noRemove := c.Request.URL.Query().Get("noRemove") == "1"
@@ -271,10 +343,10 @@ func apiReposPackageFromDir(c *gin.Context) {
return
}
dirParam := c.Params.ByName("dir")
fileParam := c.Params.ByName("file")
dirParam := utils.SanitizePath(c.Params.ByName("dir"))
fileParam := utils.SanitizePath(c.Params.ByName("file"))
if fileParam != "" && !verifyPath(fileParam) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
@@ -284,13 +356,13 @@ func apiReposPackageFromDir(c *gin.Context) {
name := c.Params.ByName("name")
repo, err := collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(repo)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -306,7 +378,7 @@ func apiReposPackageFromDir(c *gin.Context) {
resources := []string{string(repo.Key())}
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
verifier := context.GetVerifier()
var (
@@ -382,6 +454,150 @@ func apiReposPackageFromDir(c *gin.Context) {
})
}
// POST /repos/:name/copy/:src/:file
func apiReposCopyPackage(c *gin.Context) {
dstRepoName := c.Params.ByName("name")
srcRepoName := c.Params.ByName("src")
fileName := c.Params.ByName("file")
jsonBody := struct {
WithDeps bool `json:"with-deps,omitempty"`
DryRun bool `json:"dry-run,omitempty"`
}{
WithDeps: false,
DryRun: false,
}
err := c.Bind(&jsonBody)
if err != nil {
return
}
collectionFactory := context.NewCollectionFactory()
dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
return
}
err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest repo error: %s", err))
return
}
var (
srcRefList *deb.PackageRefList
srcRepo *deb.LocalRepo
)
srcRepo, err = collectionFactory.LocalRepoCollection().ByName(srcRepoName)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
return
}
if srcRepo.UUID == dstRepo.UUID {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("dest and source are identical"))
return
}
err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo)
if err != nil {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("src repo error: %s", err))
return
}
srcRefList = srcRepo.RefList()
taskName := fmt.Sprintf("Copy packages from repo %s to repo %s", srcRepoName, dstRepoName)
resources := []string{string(dstRepo.Key()), string(srcRepo.Key())}
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
reporter := &aptly.RecordingResultReporter{
Warnings: []string{},
AddedLines: []string{},
RemovedLines: []string{},
}
dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err)
}
srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err)
}
srcList.PrepareIndex()
var architecturesList []string
if jsonBody.WithDeps {
dstList.PrepareIndex()
// Calculate architectures
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = dstList.Architectures(false)
}
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to determine list of architectures, please specify explicitly")
}
}
// srcList.Filter|FilterWithProgress only accept query list
queries := make([]deb.PackageQuery, 1)
queries[0], err = query.Parse(fileName)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("unable to parse query '%s': %s", fileName, err)
}
toProcess, err := srcList.FilterWithProgress(queries, jsonBody.WithDeps, dstList, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("filter error: %s", err)
}
if toProcess.Len() == 0 {
return &task.ProcessReturnValue{Code: http.StatusUnprocessableEntity, Value: nil}, fmt.Errorf("no package found for filter: '%s'", fileName)
}
err = toProcess.ForEach(func(p *deb.Package) error {
err = dstList.Add(p)
if err != nil {
return err
}
name := fmt.Sprintf("added %s-%s(%s)", p.Name, p.Version, p.Architecture)
reporter.AddedLines = append(reporter.AddedLines, name)
return nil
})
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("error processing dest add: %s", err)
}
if jsonBody.DryRun {
reporter.Warning("Changes not saved, as dry run has been requested")
} else {
dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList))
err = collectionFactory.LocalRepoCollection().Update(dstRepo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err)
}
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{
"Report": reporter,
}}, nil
})
}
// POST /repos/:name/include/:dir/:file
func apiReposIncludePackageFromFile(c *gin.Context) {
// redirect all work to dir method
@@ -404,10 +620,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
var sources []string
var taskName string
dirParam := c.Params.ByName("dir")
fileParam := c.Params.ByName("file")
dirParam := utils.SanitizePath(c.Params.ByName("dir"))
fileParam := utils.SanitizePath(c.Params.ByName("file"))
if fileParam != "" && !verifyPath(fileParam) {
c.AbortWithError(400, fmt.Errorf("wrong file"))
AbortWithJSONError(c, 400, fmt.Errorf("wrong file"))
return
}
@@ -421,7 +637,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
repoTemplate, err := template.New("repo").Parse(repoTemplateString)
if err != nil {
c.AbortWithError(400, fmt.Errorf("error parsing repo template: %s", err))
AbortWithJSONError(c, 400, fmt.Errorf("error parsing repo template: %s", err))
return
}
@@ -432,7 +648,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
// repo template string is simple text so only use resource key of specific repository
repo, err := collectionFactory.LocalRepoCollection().ByName(repoTemplateString)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -440,7 +656,7 @@ func apiReposIncludePackageFromDir(c *gin.Context) {
}
resources = append(resources, sources...)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
var (
err error
verifier = context.GetVerifier()
api/router.go
+140 -70
View File
@@ -2,36 +2,92 @@ package api
import (
"net/http"
"os"
"sync/atomic"
"github.com/aptly-dev/aptly/aptly"
ctx "github.com/aptly-dev/aptly/context"
"github.com/aptly-dev/aptly/utils"
"github.com/gin-gonic/gin"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rs/zerolog/log"
_ "github.com/aptly-dev/aptly/docs" // import docs
swaggerFiles "github.com/swaggo/files"
ginSwagger "github.com/swaggo/gin-swagger"
)
var context *ctx.AptlyContext
func apiMetricsGet() gin.HandlerFunc {
return func(c *gin.Context) {
countPackagesByRepos()
promhttp.Handler().ServeHTTP(c.Writer, c.Request)
}
}
func redirectSwagger(c *gin.Context) {
if c.Request.URL.Path == "/docs/" {
c.Redirect(http.StatusMovedPermanently, "/docs/index.html")
return
}
c.Next()
}
// Router returns prebuilt with routes http.Handler
// @title Aptly API
// @version 1.0
// @description Aptly REST API Documentation
// @contact.name Aptly
// @contact.url http://github.com/aptly-dev/aptly
// @contact.email support@aptly.info
// @license.name MIT License
// @license.url http://www.
// @BasePath /api
func Router(c *ctx.AptlyContext) http.Handler {
context = c
router := gin.Default()
router.UseRawPath = true
router.Use(gin.ErrorLogger())
if c.Config().EnableMetricsEndpoint {
router.Use(instrumentHandlerInFlight(apiRequestsInFlightGauge, getBasePath))
router.Use(instrumentHandlerCounter(apiRequestsTotalCounter, getBasePath))
router.Use(instrumentHandlerRequestSize(apiRequestSizeSummary, getBasePath))
router.Use(instrumentHandlerResponseSize(apiResponseSizeSummary, getBasePath))
router.Use(instrumentHandlerDuration(apiRequestsDurationSummary, getBasePath))
if aptly.EnableDebug {
gin.SetMode(gin.DebugMode)
} else {
gin.SetMode(gin.ReleaseMode)
}
router := gin.New()
context = c
router.UseRawPath = true
if c.Config().LogFormat == "json" {
c.StructuredLogging(true)
utils.SetupJSONLogger(c.Config().LogLevel, os.Stdout)
gin.DefaultWriter = utils.LogWriter{Logger: log.Logger}
router.Use(JSONLogger())
} else {
c.StructuredLogging(false)
utils.SetupDefaultLogger(c.Config().LogLevel)
router.Use(gin.Logger())
}
router.Use(gin.Recovery(), gin.ErrorLogger())
if c.Config().EnableSwaggerEndpoint {
router.Use(redirectSwagger)
url := ginSwagger.URL("/docs/doc.json")
router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url))
}
if c.Config().EnableMetricsEndpoint {
MetricsCollectorRegistrar.Register(router)
}
if c.Config().ServeInAPIMode {
router.GET("/repos/", reposListInAPIMode(c.Config().FileSystemPublishRoots))
router.GET("/repos/:storage/*pkgPath", reposServeInAPIMode)
}
api := router.Group("/api")
if context.Flags().Lookup("no-lock").Value.Get().(bool) {
// We use a goroutine to count the number of
// concurrent requests. When no more requests are
@@ -40,7 +96,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
go acquireDatabase()
router.Use(func(c *gin.Context) {
api.Use(func(c *gin.Context) {
var err error
errCh := make(chan error)
@@ -48,7 +104,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
err = <-errCh
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -56,7 +112,7 @@ func Router(c *ctx.AptlyContext) http.Handler {
dbRequests <- dbRequest{releasedb, errCh}
err = <-errCh
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
}
}()
@@ -64,99 +120,113 @@ func Router(c *ctx.AptlyContext) http.Handler {
})
}
root := router.Group("/api")
{
if c.Config().EnableMetricsEndpoint {
root.GET("/metrics", apiMetricsGet())
api.GET("/metrics", apiMetricsGet())
}
root.GET("/version", apiVersion)
api.GET("/version", apiVersion)
api.GET("/storage", apiDiskFree)
isReady := &atomic.Value{}
isReady.Store(false)
defer isReady.Store(true)
api.GET("/ready", apiReady(isReady))
api.GET("/healthy", apiHealthy)
}
{
root.GET("/repos", apiReposList)
root.POST("/repos", apiReposCreate)
root.GET("/repos/:name", apiReposShow)
root.PUT("/repos/:name", apiReposEdit)
root.DELETE("/repos/:name", apiReposDrop)
api.GET("/repos", apiReposList)
api.POST("/repos", apiReposCreate)
api.GET("/repos/:name", apiReposShow)
api.PUT("/repos/:name", apiReposEdit)
api.DELETE("/repos/:name", apiReposDrop)
root.GET("/repos/:name/packages", apiReposPackagesShow)
root.POST("/repos/:name/packages", apiReposPackagesAdd)
root.DELETE("/repos/:name/packages", apiReposPackagesDelete)
api.GET("/repos/:name/packages", apiReposPackagesShow)
api.POST("/repos/:name/packages", apiReposPackagesAdd)
api.DELETE("/repos/:name/packages", apiReposPackagesDelete)
root.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
root.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
api.POST("/repos/:name/file/:dir/:file", apiReposPackageFromFile)
api.POST("/repos/:name/file/:dir", apiReposPackageFromDir)
api.POST("/repos/:name/copy/:src/:file", apiReposCopyPackage)
root.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
root.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
api.POST("/repos/:name/include/:dir/:file", apiReposIncludePackageFromFile)
api.POST("/repos/:name/include/:dir", apiReposIncludePackageFromDir)
root.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
api.POST("/repos/:name/snapshots", apiSnapshotsCreateFromRepository)
}
{
root.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
api.POST("/mirrors/:name/snapshots", apiSnapshotsCreateFromMirror)
}
{
root.GET("/mirrors", apiMirrorsList)
root.GET("/mirrors/:name", apiMirrorsShow)
root.GET("/mirrors/:name/packages", apiMirrorsPackages)
root.POST("/mirrors", apiMirrorsCreate)
root.PUT("/mirrors/:name", apiMirrorsUpdate)
root.DELETE("/mirrors/:name", apiMirrorsDrop)
api.GET("/mirrors", apiMirrorsList)
api.GET("/mirrors/:name", apiMirrorsShow)
api.GET("/mirrors/:name/packages", apiMirrorsPackages)
api.POST("/mirrors", apiMirrorsCreate)
api.PUT("/mirrors/:name", apiMirrorsUpdate)
api.DELETE("/mirrors/:name", apiMirrorsDrop)
}
{
root.POST("/gpg/key", apiGPGAddKey)
api.POST("/gpg/key", apiGPGAddKey)
}
{
root.GET("/files", apiFilesListDirs)
root.POST("/files/:dir", apiFilesUpload)
root.GET("/files/:dir", apiFilesListFiles)
root.DELETE("/files/:dir", apiFilesDeleteDir)
root.DELETE("/files/:dir/:name", apiFilesDeleteFile)
api.GET("/s3", apiS3List)
}
{
root.GET("/publish", apiPublishList)
root.POST("/publish", apiPublishRepoOrSnapshot)
root.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
root.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
root.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
api.GET("/files", apiFilesListDirs)
api.POST("/files/:dir", apiFilesUpload)
api.GET("/files/:dir", apiFilesListFiles)
api.DELETE("/files/:dir", apiFilesDeleteDir)
api.DELETE("/files/:dir/:name", apiFilesDeleteFile)
}
{
root.GET("/snapshots", apiSnapshotsList)
root.POST("/snapshots", apiSnapshotsCreate)
root.PUT("/snapshots/:name", apiSnapshotsUpdate)
root.GET("/snapshots/:name", apiSnapshotsShow)
root.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
root.DELETE("/snapshots/:name", apiSnapshotsDrop)
root.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.GET("/publish", apiPublishList)
api.POST("/publish", apiPublishRepoOrSnapshot)
api.POST("/publish/:prefix", apiPublishRepoOrSnapshot)
api.PUT("/publish/:prefix/:distribution", apiPublishUpdateSwitch)
api.DELETE("/publish/:prefix/:distribution", apiPublishDrop)
}
{
root.GET("/packages/:key", apiPackagesShow)
api.GET("/snapshots", apiSnapshotsList)
api.POST("/snapshots", apiSnapshotsCreate)
api.PUT("/snapshots/:name", apiSnapshotsUpdate)
api.GET("/snapshots/:name", apiSnapshotsShow)
api.GET("/snapshots/:name/packages", apiSnapshotsSearchPackages)
api.DELETE("/snapshots/:name", apiSnapshotsDrop)
api.GET("/snapshots/:name/diff/:withSnapshot", apiSnapshotsDiff)
api.POST("/snapshots/:name/merge", apiSnapshotsMerge)
api.POST("/snapshots/:name/pull", apiSnapshotsPull)
}
{
root.GET("/graph.:ext", apiGraph)
api.GET("/packages/:key", apiPackagesShow)
api.GET("/packages", apiPackages)
}
{
api.GET("/graph.:ext", apiGraph)
}
{
root.POST("/db/cleanup", apiDbCleanup)
api.POST("/db/cleanup", apiDbCleanup)
}
{
root.GET("/tasks", apiTasksList)
root.POST("/tasks-clear", apiTasksClear)
root.GET("/tasks-wait", apiTasksWait)
root.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
root.GET("/tasks/:id/output", apiTasksOutputShow)
root.GET("/tasks/:id/detail", apiTasksDetailShow)
root.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
root.GET("/tasks/:id", apiTasksShow)
root.DELETE("/tasks/:id", apiTasksDelete)
root.POST("/tasks-dummy", apiTasksDummy)
api.GET("/tasks", apiTasksList)
api.POST("/tasks-clear", apiTasksClear)
api.GET("/tasks-wait", apiTasksWait)
api.GET("/tasks/:id/wait", apiTasksWaitForTaskByID)
api.GET("/tasks/:id/output", apiTasksOutputShow)
api.GET("/tasks/:id/detail", apiTasksDetailShow)
api.GET("/tasks/:id/return_value", apiTasksReturnValueShow)
api.GET("/tasks/:id", apiTasksShow)
api.DELETE("/tasks/:id", apiTasksDelete)
api.POST("/tasks-dummy", apiTasksDummy)
}
return router
api/s3.go
+19
View File
@@ -0,0 +1,19 @@
package api
import (
"github.com/gin-gonic/gin"
)
// @Summary Get S3 buckets
// @Description Get list of S3 buckets.
// @Tags S3
// @Produce json
// @Success 200 {array} string "List of S3 buckets"
// @Router /api/s3 [get]
func apiS3List(c *gin.Context) {
keys := []string{}
for k := range context.Config().S3PublishRoots {
keys = append(keys, k)
}
c.JSON(200, keys)
}
api/snapshot.go
+318 -22
View File
@@ -3,15 +3,23 @@ package api
import (
"fmt"
"net/http"
"sort"
"strings"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/query"
"github.com/aptly-dev/aptly/task"
"github.com/gin-gonic/gin"
)
// GET /api/snapshots
// @Summary Get snapshots
// @Description Get list of available snapshots. Each snapshot is returned as in “show” API.
// @Tags Snapshots
// @Produce json
// @Success 200 {array} deb.Snapshot
// @Router /api/snapshots [get]
func apiSnapshotsList(c *gin.Context) {
SortMethodString := c.Request.URL.Query().Get("sort")
@@ -55,14 +63,14 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) {
repo, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
// including snapshot resource key
resources := []string{string(repo.Key()), "S" + b.Name}
taskName := fmt.Sprintf("Create snapshot of mirror %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := repo.CheckLock()
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err
@@ -123,20 +131,20 @@ func apiSnapshotsCreate(c *gin.Context) {
for i := range b.SourceSnapshots {
sources[i], err = snapshotCollection.ByName(b.SourceSnapshots[i])
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
resources = append(resources, string(sources[i].ResourceKey()))
}
maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
list := deb.NewPackageList()
// verify package refs and build package list
@@ -160,7 +168,7 @@ func apiSnapshotsCreate(c *gin.Context) {
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: nil}, nil
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
})
}
@@ -188,14 +196,14 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) {
repo, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
// including snapshot resource key
resources := []string{string(repo.Key()), "S" + b.Name}
taskName := fmt.Sprintf("Create snapshot of repo %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
err := collection.LoadComplete(repo)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
@@ -240,13 +248,13 @@ func apiSnapshotsUpdate(c *gin.Context) {
snapshot, err = collection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(snapshot.ResourceKey()), "S" + b.Name}
taskName := fmt.Sprintf("Update snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
_, err := collection.ByName(b.Name)
if err == nil {
return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)
@@ -275,13 +283,13 @@ func apiSnapshotsShow(c *gin.Context) {
snapshot, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshot)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -299,13 +307,13 @@ func apiSnapshotsDrop(c *gin.Context) {
snapshot, err := snapshotCollection.ByName(name)
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
resources := []string{string(snapshot.ResourceKey())}
taskName := fmt.Sprintf("Delete snapshot %s", name)
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
published := publishedCollection.BySnapshot(snapshot)
if len(published) > 0 {
@@ -336,32 +344,32 @@ func apiSnapshotsDiff(c *gin.Context) {
snapshotA, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
snapshotB, err := collection.ByName(c.Params.ByName("withSnapshot"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshotA)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
err = collection.LoadComplete(snapshotB)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
// Calculate diff
diff, err := snapshotA.RefList().Diff(snapshotB.RefList(), collectionFactory.PackageCollection())
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
@@ -385,15 +393,303 @@ func apiSnapshotsSearchPackages(c *gin.Context) {
snapshot, err := collection.ByName(c.Params.ByName("name"))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
err = collection.LoadComplete(snapshot)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
showPackages(c, snapshot.RefList(), collectionFactory)
}
type snapshotsMergeParams struct {
// List of snapshot names to be merged
Sources []string `binding:"required" json:"Sources" example:"snapshot1"`
}
// @Summary Snapshot Merge
// @Description **Merge several source snapshots into a new snapshot**
// @Description
// @Description Merge happens from left to right. By default, packages with the same name-architecture pair are replaced during merge (package from latest snapshot on the list wins).
// @Description
// @Description If only one snapshot is specified, merge copies source into destination.
// @Tags Snapshots
// @Param name path string true "Name of the snapshot to be created"
// @Param latest query int false "merge only the latest version of each package"
// @Param no-remove query int false "all versions of packages are preserved during merge"
// @Consume json
// @Param request body snapshotsMergeParams true "Parameters"
// @Produce json
// @Success 200
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/merge [post]
func apiSnapshotsMerge(c *gin.Context) {
var (
err error
snapshot *deb.Snapshot
body snapshotsMergeParams
)
name := c.Params.ByName("name")
if c.Bind(&body) != nil {
return
}
if len(body.Sources) < 1 {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("At least one source snapshot is required"))
return
}
latest := c.Request.URL.Query().Get("latest") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
overrideMatching := !latest && !noRemove
if noRemove && latest {
AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("no-remove and latest are mutually exclusive"))
return
}
collectionFactory := context.NewCollectionFactory()
snapshotCollection := collectionFactory.SnapshotCollection()
sources := make([]*deb.Snapshot, len(body.Sources))
resources := make([]string, len(sources))
for i := range body.Sources {
sources[i], err = snapshotCollection.ByName(body.Sources[i])
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = snapshotCollection.LoadComplete(sources[i])
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
resources[i] = string(sources[i].ResourceKey())
}
maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
result := sources[0].RefList()
for i := 1; i < len(sources); i++ {
result = result.Merge(sources[i].RefList(), overrideMatching, false)
}
if latest {
result.FilterLatestRefs()
}
sourceDescription := make([]string, len(sources))
for i, s := range sources {
sourceDescription[i] = fmt.Sprintf("'%s'", s.Name)
}
snapshot = deb.NewSnapshotFromRefList(name, sources, result,
fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", ")))
err = collectionFactory.SnapshotCollection().Add(snapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err)
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: snapshot}, nil
})
}
type snapshotsPullParams struct {
// Source name to be searched for packages and dependencies
Source string `binding:"required" json:"Source" example:"source-snapshot"`
// Name of the snapshot to be created
Destination string `binding:"required" json:"Destination" example:"idestination-snapshot"`
// List of package queries (i.e. name of package to be pulled from `Source`)
Queries []string `binding:"required" json:"Queries" example:"xserver-xorg"`
// List of architectures (optional)
Architectures []string ` json:"Architectures" example:"amd64, armhf"`
}
// @Summary Snapshot Pull
// @Description **Pulls new packages and dependencies from a source snapshot into a new snapshot**
// @Description
// @Description May also upgrade package versions if name snapshot already contains packages being pulled. New snapshot `Destination` is created as result of this process.
// @Description If architectures are limited (with config architectures or parameter `Architectures`, only mentioned architectures are processed, otherwise aptly will process all architectures in the snapshot.
// @Description If following dependencies by source is enabled (using dependencyFollowSource config), pulling binary packages would also pull corresponding source packages as well.
// @Description By default aptly would remove packages matching name and architecture while importing: e.g. when importing software_1.3_amd64, package software_1.2.9_amd64 would be removed.
// @Description
// @Description With flag `no-remove` both package versions would stay in the snapshot.
// @Description
// @Description Aptly pulls first package matching each of package queries, but with flag -all-matches all matching packages would be pulled.
// @Tags Snapshots
// @Param name path string true "Name of the snapshot to be created"
// @Param all-matches query int false "pull all the packages that satisfy the dependency version requirements (default is to pull first matching package): 1 to enable"
// @Param dry-run query int false "don’t create destination snapshot, just show what would be pulled: 1 to enable"
// @Param no-deps query int false "don’t process dependencies, just pull listed packages: 1 to enable"
// @Param no-remove query int false "don’t remove other package versions when pulling package: 1 to enable"
// @Consume json
// @Param request body snapshotsPullParams true "Parameters"
// @Produce json
// @Success 200
// @Failure 400 {object} Error "Bad Request"
// @Failure 404 {object} Error "Not Found"
// @Failure 500 {object} Error "Internal Error"
// @Router /api/snapshots/{name}/pull [post]
func apiSnapshotsPull(c *gin.Context) {
var (
err error
destinationSnapshot *deb.Snapshot
body snapshotsPullParams
)
name := c.Params.ByName("name")
if err = c.BindJSON(&body); err != nil {
AbortWithJSONError(c, http.StatusBadRequest, err)
return
}
allMatches := c.Request.URL.Query().Get("all-matches") == "1"
dryRun := c.Request.URL.Query().Get("dry-run") == "1"
noDeps := c.Request.URL.Query().Get("no-deps") == "1"
noRemove := c.Request.URL.Query().Get("no-remove") == "1"
collectionFactory := context.NewCollectionFactory()
// Load <name> snapshot
toSnapshot, err := collectionFactory.SnapshotCollection().ByName(name)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
// Load <Source> snapshot
sourceSnapshot, err := collectionFactory.SnapshotCollection().ByName(body.Source)
if err != nil {
AbortWithJSONError(c, http.StatusNotFound, err)
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot)
if err != nil {
AbortWithJSONError(c, http.StatusInternalServerError, err)
return
}
resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())}
taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination)
maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) {
// convert snapshots to package list
toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
toPackageList.PrepareIndex()
sourcePackageList.PrepareIndex()
var architecturesList []string
if len(context.ArchitecturesList()) > 0 {
architecturesList = context.ArchitecturesList()
} else {
architecturesList = toPackageList.Architectures(false)
}
architecturesList = append(architecturesList, body.Architectures...)
sort.Strings(architecturesList)
if len(architecturesList) == 0 {
err := fmt.Errorf("unable to determine list of architectures, please specify explicitly")
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Build architecture query: (arch == "i386" | arch == "amd64" | ...)
var archQuery deb.PackageQuery = &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: ""}
for _, arch := range architecturesList {
archQuery = &deb.OrQuery{L: &deb.FieldQuery{Field: "$Architecture", Relation: deb.VersionEqual, Value: arch}, R: archQuery}
}
queries := make([]deb.PackageQuery, len(body.Queries))
for i, q := range body.Queries {
queries[i], err = query.Parse(q)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
// Add architecture filter
queries[i] = &deb.AndQuery{L: queries[i], R: archQuery}
}
// Filter with dependencies as requested
destinationPackageList, err := sourcePackageList.FilterWithProgress(queries, !noDeps, toPackageList, context.DependencyOptions(), architecturesList, context.Progress())
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
destinationPackageList.PrepareIndex()
removedPackages := []string{}
addedPackages := []string{}
alreadySeen := map[string]bool{}
destinationPackageList.ForEachIndexed(func(pkg *deb.Package) error {
key := pkg.Architecture + "_" + pkg.Name
_, seen := alreadySeen[key]
// If we haven't seen such name-architecture pair and were instructed to remove, remove it
if !noRemove && !seen {
// Remove all packages with the same name and architecture
packageSearchResults := toPackageList.Search(deb.Dependency{Architecture: pkg.Architecture, Pkg: pkg.Name}, true)
for _, p := range packageSearchResults {
toPackageList.Remove(p)
removedPackages = append(removedPackages, p.String())
}
}
// If !allMatches, add only first matching name-arch package
if !seen || allMatches {
toPackageList.Add(pkg)
addedPackages = append(addedPackages, pkg.String())
}
alreadySeen[key] = true
return nil
})
alreadySeen = nil
if dryRun {
response := struct {
AddedPackages []string `json:"added_packages"`
RemovedPackages []string `json:"removed_packages"`
}{
AddedPackages: addedPackages,
RemovedPackages: removedPackages,
}
return &task.ProcessReturnValue{Code: http.StatusOK, Value: response}, nil
}
// Create <destination> snapshot
destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList,
fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", ")))
err = collectionFactory.SnapshotCollection().Add(destinationSnapshot)
if err != nil {
return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err
}
return &task.ProcessReturnValue{Code: http.StatusCreated, Value: destinationSnapshot}, nil
})
}
api/storage.go
+43
View File
@@ -0,0 +1,43 @@
package api
import (
"fmt"
"syscall"
"github.com/gin-gonic/gin"
)
type diskFree struct {
// Storage size [MiB]
Total uint64
// Available Storage [MiB]
Free uint64
// Percentage Full
PercentFull float32
}
// @Summary Get Storage Utilization
// @Description **Get disk free information of aptly storage**
// @Tags Status
// @Produce json
// @Success 200 {object} diskFree "Storage information"
// @Failure 400 {object} Error "Internal Error"
// @Router /api/storage [get]
func apiDiskFree(c *gin.Context) {
var df diskFree
fs := context.Config().GetRootDir()
var stat syscall.Statfs_t
err := syscall.Statfs(fs, &stat)
if err != nil {
AbortWithJSONError(c, 400, fmt.Errorf("Error getting storage info on %s: %s", fs, err))
return
}
df.Total = uint64(stat.Blocks) * uint64(stat.Bsize) / 1048576
df.Free = uint64(stat.Bavail) * uint64(stat.Bsize) / 1048576
df.PercentFull = 100.0 - float32(stat.Bavail)/float32(stat.Blocks)*100.0
c.JSON(200, df)
}
api/task.go
+19 -15
View File
@@ -1,7 +1,6 @@
package api
import (
"fmt"
"net/http"
"strconv"
@@ -10,7 +9,12 @@ import (
"github.com/gin-gonic/gin"
)
// GET /tasks
// @Summary Get tasks
// @Description Get list of available tasks. Each task is returned as in “show” API.
// @Tags Tasks
// @Produce json
// @Success 200 {array} task.Task
// @Router /api/tasks [get]
func apiTasksList(c *gin.Context) {
list := context.TaskList()
c.JSON(200, list.GetTasks())
@@ -35,13 +39,13 @@ func apiTasksWaitForTaskByID(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
task, err := list.WaitForTaskByID(int(id))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -53,14 +57,14 @@ func apiTasksShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var task task.Task
task, err = list.GetTaskByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -72,14 +76,14 @@ func apiTasksOutputShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var output string
output, err = list.GetTaskOutputByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -91,14 +95,14 @@ func apiTasksDetailShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var detail interface{}
detail, err = list.GetTaskDetailByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -110,13 +114,13 @@ func apiTasksReturnValueShow(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
output, err := list.GetTaskReturnValueByID(int(id))
if err != nil {
c.AbortWithError(404, err)
AbortWithJSONError(c, 404, err)
return
}
@@ -128,14 +132,14 @@ func apiTasksDelete(c *gin.Context) {
list := context.TaskList()
id, err := strconv.ParseInt(c.Params.ByName("id"), 10, 0)
if err != nil {
c.AbortWithError(500, err)
AbortWithJSONError(c, 500, err)
return
}
var delTask task.Task
delTask, err = list.DeleteTaskByID(int(id))
if err != nil {
c.AbortWithError(400, err)
AbortWithJSONError(c, 400, err)
return
}
@@ -145,7 +149,7 @@ func apiTasksDelete(c *gin.Context) {
// POST /tasks-dummy
func apiTasksDummy(c *gin.Context) {
resources := []string{"dummy"}
taskName := fmt.Sprintf("Dummy task")
taskName := "Dummy task"
maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) {
out.Printf("Dummy task started\n")
detail.Store([]int{1, 2, 3})
api/task_test.go
+1 -1
View File
@@ -71,5 +71,5 @@ func (s *TaskSuite) TestTasksClear(c *C) {
c.Check(response.Code, Equals, 200)
response, _ = s.HTTPRequest("GET", "/api/tasks", nil)
c.Check(response.Code, Equals, 200)
c.Check(response.Body.String(), Equals, "null")
c.Check(response.Body.String(), Equals, "[]")
}
aptly-api.service
-12
View File
@@ -1,12 +0,0 @@
[Unit]
Description=APT repository API
After=network.target
Documentation=man:aptly(1)
Documentation=https://www.aptly.info/doc/api/
[Service]
Type=simple
ExecStart=/usr/bin/aptly api serve -no-lock -listen=127.0.0.1:8081
[Install]
WantedBy=multi-user.target
aptly.service
-12
View File
@@ -1,12 +0,0 @@
[Unit]
Description=APT repository server
After=network.target
Documentation=man:aptly(1)
Documentation=https://www.aptly.info/doc/commands/
[Service]
Type=simple
ExecStart=/usr/bin/aptly serve -listen=127.0.0.1:8080
[Install]
WantedBy=multi-user.target
aptly/const.go
+3
View File
@@ -0,0 +1,3 @@
package aptly
var DistributionFocal = "focal"
aptly/interfaces.go
+5 -3
View File
@@ -35,8 +35,8 @@ type PackagePool interface {
Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage ChecksumStorage) (path string, err error)
// LegacyPath returns legacy (pre 1.1) path to package file (relative to root)
LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error)
// Stat returns Unix stat(2) info
Stat(path string) (os.FileInfo, error)
// Size returns the size of the given file in bytes.
Size(path string) (size int64, err error)
// Open returns ReadSeekerCloser to access the file
Open(path string) (ReadSeekerCloser, error)
// FilepathList returns file paths of all the files in the pool
@@ -47,6 +47,8 @@ type PackagePool interface {
// LocalPackagePool is implemented by PackagePools residing on the same filesystem
type LocalPackagePool interface {
// Stat returns Unix stat(2) info
Stat(path string) (os.FileInfo, error)
// GenerateTempPath generates temporary path for download (which is fast to import into package pool later on)
GenerateTempPath(filename string) (string, error)
// Link generates hardlink to destination path
@@ -70,7 +72,7 @@ type PublishedStorage interface {
// Remove removes single file under public path
Remove(path string) error
// LinkFromPool links package file from pool to dist's pool location
LinkFromPool(publishedDirectory, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool PackagePool, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error
// Filelist returns list of files under prefix
Filelist(prefix string) ([]string, error)
// RenameFile renames (moves) file
azure/azure.go
+128 -1
View File
@@ -1,2 +1,129 @@
// Package azure handles publishing to Azure Storage
package azure
// Package azure handles publishing to Azure Storage
import (
"context"
"encoding/hex"
"fmt"
"io"
"net/url"
"path/filepath"
"time"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/aptly-dev/aptly/aptly"
)
func isBlobNotFound(err error) bool {
storageError, ok := err.(azblob.StorageError)
return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound
}
type azContext struct {
container azblob.ContainerURL
prefix string
}
func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) {
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
if err != nil {
return nil, err
}
if endpoint == "" {
endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName)
}
url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container))
if err != nil {
return nil, err
}
containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
result := &azContext{
container: containerURL,
prefix: prefix,
}
return result, nil
}
func (az *azContext) blobPath(path string) string {
return filepath.Join(az.prefix, path)
}
func (az *azContext) blobURL(path string) azblob.BlobURL {
return az.container.NewBlobURL(az.blobPath(path))
}
func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) {
const delimiter = "/"
paths = make([]string, 0, 1024)
md5s = make([]string, 0, 1024)
prefix = filepath.Join(az.prefix, prefix)
if prefix != "" {
prefix += delimiter
}
for marker := (azblob.Marker{}); marker.NotDone(); {
listBlob, err := az.container.ListBlobsFlatSegment(
context.Background(), marker, azblob.ListBlobsSegmentOptions{
Prefix: prefix,
MaxResults: 1,
Details: azblob.BlobListingDetails{Metadata: true}})
if err != nil {
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err)
}
marker = listBlob.NextMarker
for _, blob := range listBlob.Segment.BlobItems {
if prefix == "" {
paths = append(paths, blob.Name)
} else {
paths = append(paths, blob.Name[len(prefix):])
}
md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
}
if progress != nil {
time.Sleep(time.Duration(500) * time.Millisecond)
progress.AddBar(1)
}
}
return paths, md5s, nil
}
func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error {
uploadOptions := azblob.UploadStreamToBlockBlobOptions{
BufferSize: 4 * 1024 * 1024,
MaxBuffers: 8,
}
if len(sourceMD5) > 0 {
decodedMD5, err := hex.DecodeString(sourceMD5)
if err != nil {
return err
}
uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
ContentMD5: decodedMD5,
}
}
_, err := azblob.UploadStreamToBlockBlob(
context.Background(),
source,
blob.ToBlockBlobURL(),
uploadOptions,
)
return err
}
// String
func (az *azContext) String() string {
return fmt.Sprintf("Azure: %s/%s", az.container, az.prefix)
}
azure/package_pool.go
+218
View File
@@ -0,0 +1,218 @@
package azure
import (
"context"
"os"
"path/filepath"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/pkg/errors"
)
type PackagePool struct {
az *azContext
}
// Check interface
var (
_ aptly.PackagePool = (*PackagePool)(nil)
)
// NewPackagePool creates published storage from Azure storage credentials
func NewPackagePool(accountName, accountKey, container, prefix, endpoint string) (*PackagePool, error) {
azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
if err != nil {
return nil, err
}
return &PackagePool{az: azctx}, nil
}
// String
func (pool *PackagePool) String() string {
return pool.az.String()
}
func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.ChecksumInfo) string {
hash := checksums.SHA256
// Use the same path as the file pool, for compat reasons.
return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename)
}
func (pool *PackagePool) ensureChecksums(
poolPath string,
checksumStorage aptly.ChecksumStorage,
) (*utils.ChecksumInfo, error) {
targetChecksums, err := checksumStorage.Get(poolPath)
if err != nil {
return nil, err
}
if targetChecksums == nil {
// we don't have checksums stored yet for this file
blob := pool.az.blobURL(poolPath)
download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
if err != nil {
if isBlobNotFound(err) {
return nil, nil
}
return nil, errors.Wrapf(err, "error downloading blob at %s", poolPath)
}
targetChecksums = &utils.ChecksumInfo{}
*targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{}))
if err != nil {
return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath)
}
err = checksumStorage.Update(poolPath, targetChecksums)
if err != nil {
return nil, err
}
}
return targetChecksums, nil
}
func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) {
if progress != nil {
progress.InitBar(0, false, aptly.BarGeneralBuildFileList)
defer progress.ShutdownBar()
}
paths, _, err := pool.az.internalFilelist("", progress)
return paths, err
}
func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, error) {
return "", errors.New("Azure package pool does not support legacy paths")
}
func (pool *PackagePool) Size(path string) (int64, error) {
blob := pool.az.blobURL(path)
props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
if err != nil {
return 0, errors.Wrapf(err, "error examining %s from %s", path, pool)
}
return props.ContentLength(), nil
}
func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) {
blob := pool.az.blobURL(path)
temp, err := os.CreateTemp("", "blob-download")
if err != nil {
return nil, errors.Wrap(err, "error creating temporary file for blob download")
}
defer os.Remove(temp.Name())
err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{})
if err != nil {
return nil, errors.Wrapf(err, "error downloading blob at %s", path)
}
return temp, nil
}
func (pool *PackagePool) Remove(path string) (int64, error) {
blob := pool.az.blobURL(path)
props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
if err != nil {
return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool)
}
_, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
if err != nil {
return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool)
}
return props.ContentLength(), nil
}
func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) {
if checksums.MD5 == "" || checksums.SHA256 == "" || checksums.SHA512 == "" {
// need to update checksums, MD5 and SHA256 should be always defined
var err error
*checksums, err = utils.ChecksumsForFile(srcPath)
if err != nil {
return "", err
}
}
path := pool.buildPoolPath(basename, checksums)
blob := pool.az.blobURL(path)
targetChecksums, err := pool.ensureChecksums(path, checksumStorage)
if err != nil {
return "", err
} else if targetChecksums != nil {
// target already exists
*checksums = *targetChecksums
return path, nil
}
source, err := os.Open(srcPath)
if err != nil {
return "", err
}
defer source.Close()
err = pool.az.putFile(blob, source, checksums.MD5)
if err != nil {
return "", err
}
if !checksums.Complete() {
// need full checksums here
*checksums, err = utils.ChecksumsForFile(srcPath)
if err != nil {
return "", err
}
}
err = checksumStorage.Update(path, checksums)
if err != nil {
return "", err
}
return path, nil
}
func (pool *PackagePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) {
if poolPath == "" {
if checksums.SHA256 != "" {
poolPath = pool.buildPoolPath(basename, checksums)
} else {
// No checksums or pool path, so no idea what file to look for.
return "", false, nil
}
}
size, err := pool.Size(poolPath)
if err != nil {
return "", false, err
} else if size != checksums.Size {
return "", false, nil
}
targetChecksums, err := pool.ensureChecksums(poolPath, checksumStorage)
if err != nil {
return "", false, err
} else if targetChecksums == nil {
return "", false, nil
}
if checksums.MD5 != "" && targetChecksums.MD5 != checksums.MD5 ||
checksums.SHA256 != "" && targetChecksums.SHA256 != checksums.SHA256 {
// wrong file?
return "", false, nil
}
// fill back checksums
*checksums = *targetChecksums
return poolPath, true, nil
}
azure/package_pool_test.go
+255
View File
@@ -0,0 +1,255 @@
package azure
import (
"context"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/files"
"github.com/aptly-dev/aptly/utils"
. "gopkg.in/check.v1"
)
type PackagePoolSuite struct {
accountName, accountKey, endpoint string
pool, prefixedPool *PackagePool
debFile string
cs aptly.ChecksumStorage
}
var _ = Suite(&PackagePoolSuite{})
func (s *PackagePoolSuite) SetUpSuite(c *C) {
s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
if s.accountName == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCOUNT not set.")
}
s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
if s.accountKey == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
}
s.endpoint = os.Getenv("AZURE_STORAGE_ENDPOINT")
}
func (s *PackagePoolSuite) SetUpTest(c *C) {
container := randContainer()
prefix := "lala"
var err error
s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
cnt := s.pool.az.container
_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
c.Assert(err, IsNil)
s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint)
c.Assert(err, IsNil)
_, _File, _, _ := runtime.Caller(0)
s.debFile = filepath.Join(filepath.Dir(_File), "../system/files/libboost-program-options-dev_1.49.0.1_i386.deb")
s.cs = files.NewMockChecksumStorage()
}
func (s *PackagePoolSuite) TestFilepathList(c *C) {
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{})
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
list, err = s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{
"c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb",
"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb",
})
}
func (s *PackagePoolSuite) TestRemove(c *C) {
s.pool.Import(s.debFile, "a.deb", &utils.ChecksumInfo{}, false, s.cs)
s.pool.Import(s.debFile, "b.deb", &utils.ChecksumInfo{}, false, s.cs)
size, err := s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
c.Check(err, IsNil)
c.Check(size, Equals, int64(2738))
_, err = s.pool.Remove("c7/6b/4bd12fd92e4dfe1b55b18a67a669_a.deb")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
list, err := s.pool.FilepathList(nil)
c.Check(err, IsNil)
c.Check(list, DeepEquals, []string{"c7/6b/4bd12fd92e4dfe1b55b18a67a669_b.deb"})
}
func (s *PackagePoolSuite) TestImportOk(c *C) {
var checksum utils.ChecksumInfo
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// SHA256 should be automatically calculated
c.Check(checksum.SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
// checksum storage is filled with new checksum
c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
size, err := s.pool.Size(path)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(2738))
// import as different name
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, "some.deb", &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_some.deb")
// checksum storage is filled with new checksum
c.Check(s.cs.(*files.MockChecksumStorage).Store[path].SHA256, Equals, "c76b4bd12fd92e4dfe1b55b18a67a669d92f62985d6a96c8a21d96120982cf12")
// double import, should be ok
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// clear checksum storage, and do double-import
delete(s.cs.(*files.MockChecksumStorage).Store, path)
checksum = utils.ChecksumInfo{}
path, err = s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// checksum is filled back based on re-calculation of file in the pool
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// import under new name, but with path-relevant checksums already filled in
checksum = utils.ChecksumInfo{SHA256: checksum.SHA256}
path, err = s.pool.Import(s.debFile, "other.deb", &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_other.deb")
// checksum is filled back based on re-calculation of source file
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
}
func (s *PackagePoolSuite) TestVerify(c *C) {
// file doesn't exist yet
ppath, exists, err := s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// import file
checksum := utils.ChecksumInfo{}
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &checksum, false, s.cs)
c.Check(err, IsNil)
c.Check(path, Equals, "c7/6b/4bd12fd92e4dfe1b55b18a67a669_libboost-program-options-dev_1.49.0.1_i386.deb")
// check existence
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, ppath)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence with fixed path
checksum = utils.ChecksumInfo{Size: checksum.Size}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, but with checksums missing (that aren't needed to find the path)
checksum.SHA512 = ""
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with missing checksum info but correct path and size available
checksum = utils.ChecksumInfo{Size: checksum.Size}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on checksum storage
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with wrong checksum info but correct path and size available
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &utils.ChecksumInfo{
SHA256: "abc",
Size: checksum.Size,
}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// check existence, with missing checksums (that aren't needed to find the path)
// and no info in checksum storage
delete(s.cs.(*files.MockChecksumStorage).Store, path)
checksum.SHA512 = ""
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, path)
c.Check(err, IsNil)
c.Check(exists, Equals, true)
// checksum is filled back based on re-calculation
c.Check(checksum.SHA512, Equals, "d7302241373da972aa9b9e71d2fd769b31a38f71182aa71bc0d69d090d452c69bb74b8612c002ccf8a89c279ced84ac27177c8b92d20f00023b3d268e6cec69c")
// check existence, with wrong size
checksum = utils.ChecksumInfo{Size: 13455}
ppath, exists, err = s.pool.Verify(path, filepath.Base(s.debFile), &checksum, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
// check existence, with empty checksum info
ppath, exists, err = s.pool.Verify("", filepath.Base(s.debFile), &utils.ChecksumInfo{}, s.cs)
c.Check(ppath, Equals, "")
c.Check(err, IsNil)
c.Check(exists, Equals, false)
}
func (s *PackagePoolSuite) TestImportNotExist(c *C) {
_, err := s.pool.Import("no-such-file", "a.deb", &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, ErrorMatches, ".*no such file or directory")
}
func (s *PackagePoolSuite) TestSize(c *C) {
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, IsNil)
size, err := s.pool.Size(path)
c.Assert(err, IsNil)
c.Check(size, Equals, int64(2738))
_, err = s.pool.Size("do/es/ntexist")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
}
func (s *PackagePoolSuite) TestOpen(c *C) {
path, err := s.pool.Import(s.debFile, filepath.Base(s.debFile), &utils.ChecksumInfo{}, false, s.cs)
c.Check(err, IsNil)
f, err := s.pool.Open(path)
c.Assert(err, IsNil)
contents, err := ioutil.ReadAll(f)
c.Assert(err, IsNil)
c.Check(len(contents), Equals, 2738)
c.Check(f.Close(), IsNil)
_, err = s.pool.Open("do/es/ntexist")
c.Check(err, ErrorMatches, "(.|\n)*BlobNotFound(.|\n)*")
}
azure/public.go
+34 -131
View File
@@ -2,12 +2,8 @@ package azure
import (
"context"
"encoding/hex"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"time"
@@ -22,7 +18,8 @@ import (
type PublishedStorage struct {
container azblob.ContainerURL
prefix string
pathCache map[string]string
az *azContext
pathCache map[string]map[string]string
}
// Check interface
@@ -30,60 +27,23 @@ var (
_ aptly.PublishedStorage = (*PublishedStorage)(nil)
)
func isEmulatorEndpoint(endpoint string) bool {
if h, _, err := net.SplitHostPort(endpoint); err == nil {
endpoint = h
}
if endpoint == "localhost" {
return true
}
// For IPv6, there could be case where SplitHostPort fails for cannot finding port.
// In this case, eliminate the '[' and ']' in the URL.
// For details about IPv6 URL, please refer to https://tools.ietf.org/html/rfc2732
if endpoint[0] == '[' && endpoint[len(endpoint)-1] == ']' {
endpoint = endpoint[1 : len(endpoint)-1]
}
return net.ParseIP(endpoint) != nil
}
// NewPublishedStorage creates published storage from Azure storage credentials
func NewPublishedStorage(accountName, accountKey, container, prefix, endpoint string) (*PublishedStorage, error) {
credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
azctx, err := newAzContext(accountName, accountKey, container, prefix, endpoint)
if err != nil {
return nil, err
}
if endpoint == "" {
endpoint = "blob.core.windows.net"
}
var url *url.URL
if isEmulatorEndpoint(endpoint) {
url, err = url.Parse(fmt.Sprintf("http://%s/%s/%s", endpoint, accountName, container))
} else {
url, err = url.Parse(fmt.Sprintf("https://%s.%s/%s", accountName, endpoint, container))
}
if err != nil {
return nil, err
}
containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{}))
result := &PublishedStorage{
container: containerURL,
prefix: prefix,
}
return result, nil
return &PublishedStorage{az: azctx}, nil
}
// String
func (storage *PublishedStorage) String() string {
return fmt.Sprintf("Azure: %s/%s", storage.container, storage.prefix)
return storage.az.String()
}
// MkDir creates directory recursively under public path
func (storage *PublishedStorage) MkDir(path string) error {
func (storage *PublishedStorage) MkDir(_ string) error {
// no op for Azure
return nil
}
@@ -106,7 +66,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
}
defer source.Close()
err = storage.putFile(path, source, sourceMD5)
err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5)
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage))
}
@@ -114,45 +74,15 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err
return err
}
// putFile uploads file-like object to
func (storage *PublishedStorage) putFile(path string, source io.Reader, sourceMD5 string) error {
path = filepath.Join(storage.prefix, path)
blob := storage.container.NewBlockBlobURL(path)
uploadOptions := azblob.UploadStreamToBlockBlobOptions{
BufferSize: 4 * 1024 * 1024,
MaxBuffers: 8,
}
if len(sourceMD5) > 0 {
decodedMD5, err := hex.DecodeString(sourceMD5)
if err != nil {
return err
}
uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{
ContentMD5: decodedMD5,
}
}
_, err := azblob.UploadStreamToBlockBlob(
context.Background(),
source,
blob,
uploadOptions,
)
return err
}
// RemoveDirs removes directory structure under public path
func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress) error {
func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error {
filelist, err := storage.Filelist(path)
if err != nil {
return err
}
for _, filename := range filelist {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path, filename))
blob := storage.az.blobURL(filepath.Join(path, filename))
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
if err != nil {
return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err)
@@ -164,7 +94,7 @@ func (storage *PublishedStorage) RemoveDirs(path string, progress aptly.Progress
// Remove removes single file under public path
func (storage *PublishedStorage) Remove(path string) error {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
blob := storage.az.blobURL(path)
_, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
if err != nil {
err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err))
@@ -174,31 +104,39 @@ func (storage *PublishedStorage) Remove(path string) error {
// LinkFromPool links package file from pool to dist's pool location
//
// publishedDirectory is desired location in pool (like prefix/pool/component/liba/libav/)
// publishedPrefix is desired prefix for the location in the pool.
// publishedRelPath is desired location in pool (like pool/component/liba/libav/)
// sourcePool is instance of aptly.PackagePool
// sourcePath is filepath to package file in package pool
//
// LinkFromPool returns relative path for the published file to be included in package index
func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName string, sourcePool aptly.PackagePool,
func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, fileName string, sourcePool aptly.PackagePool,
sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error {
relPath := filepath.Join(publishedDirectory, fileName)
poolPath := filepath.Join(storage.prefix, relPath)
relFilePath := filepath.Join(publishedRelPath, fileName)
// prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath)
// FIXME: check how to integrate publishedPrefix:
poolPath := storage.az.blobPath(fileName)
if storage.pathCache == nil {
paths, md5s, err := storage.internalFilelist("")
storage.pathCache = make(map[string]map[string]string)
}
pathCache := storage.pathCache[publishedPrefix]
if pathCache == nil {
paths, md5s, err := storage.az.internalFilelist(publishedPrefix, nil)
if err != nil {
return fmt.Errorf("error caching paths under prefix: %s", err)
}
storage.pathCache = make(map[string]string, len(paths))
pathCache = make(map[string]string, len(paths))
for i := range paths {
storage.pathCache[paths[i]] = md5s[i]
pathCache[paths[i]] = md5s[i]
}
storage.pathCache[publishedPrefix] = pathCache
}
destinationMD5, exists := storage.pathCache[relPath]
destinationMD5, exists := pathCache[relFilePath]
sourceMD5 := sourceChecksums.MD5
if exists {
@@ -221,9 +159,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
}
defer source.Close()
err = storage.putFile(relPath, source, sourceMD5)
err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5)
if err == nil {
storage.pathCache[relPath] = sourceMD5
pathCache[relFilePath] = sourceMD5
} else {
err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath))
}
@@ -231,43 +169,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin
return err
}
func (storage *PublishedStorage) internalFilelist(prefix string) (paths []string, md5s []string, err error) {
const delimiter = "/"
paths = make([]string, 0, 1024)
md5s = make([]string, 0, 1024)
prefix = filepath.Join(storage.prefix, prefix)
if prefix != "" {
prefix += delimiter
}
for marker := (azblob.Marker{}); marker.NotDone(); {
listBlob, err := storage.container.ListBlobsFlatSegment(
context.Background(), marker, azblob.ListBlobsSegmentOptions{
Prefix: prefix,
MaxResults: 1000,
Details: azblob.BlobListingDetails{Metadata: true}})
if err != nil {
return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, storage, err)
}
marker = listBlob.NextMarker
for _, blob := range listBlob.Segment.BlobItems {
if prefix == "" {
paths = append(paths, blob.Name)
} else {
paths = append(paths, blob.Name[len(prefix):])
}
md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5))
}
}
return paths, md5s, nil
}
// Filelist returns list of files under prefix
func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
paths, _, err := storage.internalFilelist(prefix)
paths, _, err := storage.az.internalFilelist(prefix, nil)
return paths, err
}
@@ -275,8 +179,8 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) {
func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error {
const leaseDuration = 30
dstBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, dst))
srcBlobURL := storage.container.NewBlobURL(filepath.Join(storage.prefix, src))
dstBlobURL := storage.az.blobURL(dst)
srcBlobURL := storage.az.blobURL(src)
leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{})
if err != nil || leaseResp.StatusCode() != http.StatusCreated {
return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL)
@@ -347,11 +251,10 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error {
// FileExists returns true if path exists
func (storage *PublishedStorage) FileExists(path string) (bool, error) {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
blob := storage.az.blobURL(path)
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
if err != nil {
storageError, ok := err.(azblob.StorageError)
if ok && string(storageError.ServiceCode()) == string(azblob.StorageErrorCodeBlobNotFound) {
if isBlobNotFound(err) {
return false, nil
}
return false, err
@@ -364,7 +267,7 @@ func (storage *PublishedStorage) FileExists(path string) (bool, error) {
// ReadLink returns the symbolic link pointed to by path.
// This simply reads text file created with SymLink
func (storage *PublishedStorage) ReadLink(path string) (string, error) {
blob := storage.container.NewBlobURL(filepath.Join(storage.prefix, path))
blob := storage.az.blobURL(path)
resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
if err != nil {
return "", err
azure/public_test.go
+15 -15
View File
@@ -43,14 +43,14 @@ func randString(n int) string {
func (s *PublishedStorageSuite) SetUpSuite(c *C) {
s.accountName = os.Getenv("AZURE_STORAGE_ACCOUNT")
if s.accountName == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println("Please set the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCOUNT not set.")
}
s.accountKey = os.Getenv("AZURE_STORAGE_ACCESS_KEY")
if s.accountKey == "" {
println("Please set the the following two environment variables to run the Azure storage tests.")
println("Please set the following two environment variables to run the Azure storage tests.")
println(" 1. AZURE_STORAGE_ACCOUNT")
println(" 2. AZURE_STORAGE_ACCESS_KEY")
c.Skip("AZURE_STORAGE_ACCESS_KEY not set.")
@@ -66,7 +66,7 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint)
c.Assert(err, IsNil)
cnt := s.storage.container
cnt := s.storage.az.container
_, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer)
c.Assert(err, IsNil)
@@ -75,13 +75,13 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) {
}
func (s *PublishedStorageSuite) TearDownTest(c *C) {
cnt := s.storage.container
cnt := s.storage.az.container
_, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{})
c.Assert(err, IsNil)
}
func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
blob := s.storage.container.NewBlobURL(path)
blob := s.storage.az.container.NewBlobURL(path)
resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
c.Assert(err, IsNil)
body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3})
@@ -91,7 +91,7 @@ func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte {
}
func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) {
_, err := s.storage.container.NewBlobURL(path).GetProperties(
_, err := s.storage.az.container.NewBlobURL(path).GetProperties(
context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{})
c.Assert(err, NotNil)
storageError, ok := err.(azblob.StorageError)
@@ -104,7 +104,7 @@ func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) {
_, err := azblob.UploadBufferToBlockBlob(
context.Background(),
data,
s.storage.container.NewBlockBlobURL(path),
s.storage.az.container.NewBlockBlobURL(path),
azblob.UploadToBlockBlobOptions{
BlobHTTPHeaders: azblob.BlobHTTPHeaders{
ContentMD5: hash[:],
@@ -129,7 +129,7 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) {
err = s.prefixedStorage.PutFile(filename, filepath.Join(dir, "a"))
c.Check(err, IsNil)
c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.prefix, filename)), DeepEquals, content)
c.Check(s.GetFile(c, filepath.Join(s.prefixedStorage.az.prefix, filename)), DeepEquals, content)
}
func (s *PublishedStorageSuite) TestPutFilePlus(c *C) {
@@ -300,45 +300,45 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) {
c.Assert(err, IsNil)
// first link from pool
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.storage.LinkFromPool("", filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// duplicate link from pool
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, false)
c.Check(err, ErrorMatches, ".*file already exists and is different.*")
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with conflict and force
err = s.storage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
err = s.storage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src2, cksum2, true)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Spam"))
// for prefixed storage:
// first link from pool
err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, src1, cksum1, false)
c.Check(err, IsNil)
// 2nd link from pool, providing wrong path for source file
//
// this test should check that file already exists in S3 and skip upload (which would fail if not skipped)
s.prefixedStorage.pathCache = nil
err = s.prefixedStorage.LinkFromPool(filepath.Join("", "pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "lala/pool/main/m/mars-invaders/mars-invaders_1.03.deb"), DeepEquals, []byte("Contents"))
// link from pool with nested file name
err = s.storage.LinkFromPool("dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
err = s.storage.LinkFromPool("", "dists/jessie/non-free/installer-i386/current/images", "netboot/boot.img.gz", pool, src3, cksum3, false)
c.Check(err, IsNil)
c.Check(s.GetFile(c, "dists/jessie/non-free/installer-i386/current/images/netboot/boot.img.gz"), DeepEquals, []byte("Contents"))
cmd/api_serve.go
+22 -9
View File
@@ -1,11 +1,15 @@
package cmd
import (
stdcontext "context"
"errors"
"fmt"
"net"
"net/http"
"net/url"
"os"
"os/signal"
"syscall"
"github.com/aptly-dev/aptly/api"
"github.com/aptly-dev/aptly/systemd/activation"
@@ -30,7 +34,7 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
// anything else must fail.
// E.g.: Running the service under a different user may lead to a rootDir
// that exists but is not usable due to access permissions.
err = utils.DirIsAccessible(context.Config().RootDir)
err = utils.DirIsAccessible(context.Config().GetRootDir())
if err != nil {
return err
}
@@ -55,6 +59,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
listen := context.Flags().Lookup("listen").Value.String()
fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen)
server := http.Server{Handler: api.Router(context)}
sigchan := make(chan os.Signal, 1)
signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM)
go (func() {
if _, ok := <-sigchan; ok {
server.Shutdown(stdcontext.Background())
}
})()
defer close(sigchan)
listenURL, err := url.Parse(listen)
if err == nil && listenURL.Scheme == "unix" {
file := listenURL.Path
@@ -67,19 +82,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error {
}
defer listener.Close()
err = http.Serve(listener, api.Router(context))
if err != nil {
return fmt.Errorf("unable to serve: %s", err)
}
return nil
err = server.Serve(listener)
} else {
server.Addr = listen
err = server.ListenAndServe()
}
err = http.ListenAndServe(listen, api.Router(context))
if err != nil {
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return fmt.Errorf("unable to serve: %s", err)
}
return err
return nil
}
func makeCmdAPIServe() *commander.Command {
cmd/cmd.go
+1 -1
View File
@@ -118,7 +118,7 @@ package environment to new version.`,
cmd.Flag.Bool("dep-follow-all-variants", false, "when processing dependencies, follow a & b if dependency is 'a|b'")
cmd.Flag.Bool("dep-verbose-resolve", false, "when processing dependencies, print detailed logs")
cmd.Flag.String("architectures", "", "list of architectures to consider during (comma-separated), default to all available")
cmd.Flag.String("config", "", "location of configuration file (default locations are /etc/aptly.conf, ~/.aptly.conf)")
cmd.Flag.String("config", "", "location of configuration file (default locations in order: ~/.aptly.conf, /usr/local/etc/aptly.conf, /etc/aptly.conf)")
cmd.Flag.String("gpg-provider", "", "PGP implementation (\"gpg\", \"gpg1\", \"gpg2\" for external gpg or \"internal\" for Go internal implementation)")
if aptly.EnableDebug {
cmd/config_show.go
+1 -1
View File
@@ -7,7 +7,7 @@ import (
"github.com/smira/commander"
)
func aptlyConfigShow(cmd *commander.Command, args []string) error {
func aptlyConfigShow(_ *commander.Command, _ []string) error {
config := context.Config()
prettyJSON, err := json.MarshalIndent(config, "", " ")
cmd/graph.go
+1 -2
View File
@@ -4,7 +4,6 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
@@ -36,7 +35,7 @@ func aptlyGraph(cmd *commander.Command, args []string) error {
buf := bytes.NewBufferString(graph.String())
tempfile, err := ioutil.TempFile("", "aptly-graph")
tempfile, err := os.CreateTemp("", "aptly-graph")
if err != nil {
return err
}
cmd/mirror.go
+5 -5
View File
@@ -9,18 +9,18 @@ import (
)
func getVerifier(flags *flag.FlagSet) (pgp.Verifier, error) {
if LookupOption(context.Config().GpgDisableVerify, flags, "ignore-signatures") {
return nil, nil
}
keyRings := flags.Lookup("keyring").Value.Get().([]string)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
verifier := context.GetVerifier()
for _, keyRing := range keyRings {
verifier.AddKeyring(keyRing)
}
err := verifier.InitKeyring()
err := verifier.InitKeyring(ignoreSignatures == false) // be verbose only if verifying signatures is requested
if err != nil {
return nil, err
}
cmd/mirror_create.go
+5 -1
View File
@@ -20,6 +20,10 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
downloadSources := LookupOption(context.Config().DownloadSourcePackages, context.Flags(), "with-sources")
downloadUdebs := context.Flags().Lookup("with-udebs").Value.Get().(bool)
downloadInstaller := context.Flags().Lookup("with-installer").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
var (
mirrorName, archiveURL, distribution string
@@ -59,7 +63,7 @@ func aptlyMirrorCreate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to fetch mirror: %s", err)
}
cmd/mirror_edit.go
+4 -1
View File
@@ -28,6 +28,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
}
fetchMirror := false
ignoreSignatures := context.Config().GpgDisableVerify
context.Flags().Visit(func(flag *flag.Flag) {
switch flag.Name {
case "filter":
@@ -43,6 +44,8 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
case "archive-url":
repo.SetArchiveRoot(flag.Value.String())
fetchMirror = true
case "ignore-signatures":
ignoreSignatures = true
}
})
@@ -69,7 +72,7 @@ func aptlyMirrorEdit(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to edit: %s", err)
}
cmd/mirror_list.go
+2 -2
View File
@@ -24,7 +24,7 @@ func aptlyMirrorList(cmd *commander.Command, args []string) error {
return aptlyMirrorListTxt(cmd, args)
}
func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
func aptlyMirrorListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -65,7 +65,7 @@ func aptlyMirrorListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyMirrorListJSON(cmd *commander.Command, args []string) error {
func aptlyMirrorListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.RemoteRepo, context.NewCollectionFactory().RemoteRepoCollection().Len())
cmd/mirror_show.go
+2 -2
View File
@@ -27,7 +27,7 @@ func aptlyMirrorShow(cmd *commander.Command, args []string) error {
return aptlyMirrorShowTxt(cmd, args)
}
func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
func aptlyMirrorShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
@@ -93,7 +93,7 @@ func aptlyMirrorShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyMirrorShowJSON(cmd *commander.Command, args []string) error {
func aptlyMirrorShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/mirror_update.go
+31 -5
View File
@@ -2,6 +2,7 @@ package cmd
import (
"fmt"
"os"
"strings"
"sync"
@@ -41,20 +42,24 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
}
}
ignoreMismatch := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
ignoreChecksums := context.Flags().Lookup("ignore-checksums").Value.Get().(bool)
verifier, err := getVerifier(context.Flags())
if err != nil {
return fmt.Errorf("unable to initialize GPG verifier: %s", err)
}
err = repo.Fetch(context.Downloader(), verifier)
err = repo.Fetch(context.Downloader(), verifier, ignoreSignatures)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
context.Progress().Printf("Downloading & parsing package files...\n")
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreMismatch)
err = repo.DownloadPackageIndexes(context.Progress(), context.Downloader(), verifier, collectionFactory, ignoreSignatures, ignoreChecksums)
if err != nil {
return fmt.Errorf("unable to update: %s", err)
}
@@ -161,7 +166,16 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
var e error
// provision download location
task.TempDownPath, e = context.PackagePool().(aptly.LocalPackagePool).GenerateTempPath(task.File.Filename)
if pp, ok := context.PackagePool().(aptly.LocalPackagePool); ok {
task.TempDownPath, e = pp.GenerateTempPath(task.File.Filename)
} else {
var file *os.File
file, e = os.CreateTemp("", task.File.Filename)
if e == nil {
task.TempDownPath = file.Name()
file.Close()
}
}
if e != nil {
pushError(e)
continue
@@ -173,7 +187,7 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
repo.PackageURL(task.File.DownloadURL()).String(),
task.TempDownPath,
&task.File.Checksums,
ignoreMismatch)
ignoreChecksums)
if e != nil {
pushError(e)
continue
@@ -197,6 +211,18 @@ func aptlyMirrorUpdate(cmd *commander.Command, args []string) error {
return fmt.Errorf("unable to update: %s", err)
}
defer func() {
for _, task := range queue {
if task.TempDownPath == "" {
continue
}
if err := os.Remove(task.TempDownPath); err != nil && !os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "Failed to delete %s: %v\n", task.TempDownPath, err)
}
}
}()
// Import downloaded files
context.Progress().InitBar(int64(len(queue)), false, aptly.BarMirrorUpdateImportFiles)
cmd/publish_list.go
+3 -3
View File
@@ -25,7 +25,7 @@ func aptlyPublishList(cmd *commander.Command, args []string) error {
return aptlyPublishListTxt(cmd, args)
}
func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
func aptlyPublishListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -34,7 +34,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
published := make([]string, 0, collectionFactory.PublishedRepoCollection().Len())
err = collectionFactory.PublishedRepoCollection().ForEach(func(repo *deb.PublishedRepo) error {
e := collectionFactory.PublishedRepoCollection().LoadComplete(repo, collectionFactory)
e := collectionFactory.PublishedRepoCollection().LoadShallow(repo, collectionFactory)
if e != nil {
fmt.Fprintf(os.Stderr, "Error found on one publish (prefix:%s / distribution:%s / component:%s\n)",
repo.StoragePrefix(), repo.Distribution, repo.Components())
@@ -77,7 +77,7 @@ func aptlyPublishListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyPublishListJSON(cmd *commander.Command, args []string) error {
func aptlyPublishListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.PublishedRepo, 0, context.NewCollectionFactory().PublishedRepoCollection().Len())
cmd/publish_repo.go
+2
View File
@@ -48,8 +48,10 @@ Example:
cmd.Flag.String("butautomaticupgrades", "", "set value for ButAutomaticUpgrades field")
cmd.Flag.String("label", "", "label to publish")
cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_show.go
+2 -2
View File
@@ -24,7 +24,7 @@ func aptlyPublishShow(cmd *commander.Command, args []string) error {
return aptlyPublishShowTxt(cmd, args)
}
func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
func aptlyPublishShowTxt(_ *commander.Command, args []string) error {
var err error
distribution := args[0]
@@ -76,7 +76,7 @@ func aptlyPublishShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyPublishShowJSON(cmd *commander.Command, args []string) error {
func aptlyPublishShowJSON(_ *commander.Command, args []string) error {
var err error
distribution := args[0]
cmd/publish_snapshot.go
+6 -3
View File
@@ -116,8 +116,9 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
origin := context.Flags().Lookup("origin").Value.String()
notAutomatic := context.Flags().Lookup("notautomatic").Value.String()
butAutomaticUpgrades := context.Flags().Lookup("butautomaticupgrades").Value.String()
multiDist := context.Flags().Lookup("multi-dist").Value.Get().(bool)
published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory)
published, err := deb.NewPublishedRepo(storage, prefix, distribution, context.ArchitecturesList(), components, sources, collectionFactory, multiDist)
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
}
@@ -132,6 +133,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
}
published.Label = context.Flags().Lookup("label").Value.String()
published.Suite = context.Flags().Lookup("suite").Value.String()
published.Codename = context.Flags().Lookup("codename").Value.String()
published.SkipContents = context.Config().SkipContentsPublishing
@@ -161,8 +163,7 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error {
forceOverwrite := context.Flags().Lookup("force-overwrite").Value.Get().(bool)
if forceOverwrite {
context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing " +
"the same package pool.\n")
context.Progress().ColoredPrintf("@rWARNING@|: force overwrite mode enabled, aptly might corrupt other published repositories sharing the same package pool.\n")
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
@@ -239,8 +240,10 @@ Example:
cmd.Flag.String("butautomaticupgrades", "", "overwrite value for ButAutomaticUpgrades field")
cmd.Flag.String("label", "", "label to publish")
cmd.Flag.String("suite", "", "suite to publish (defaults to distribution)")
cmd.Flag.String("codename", "", "codename to publish (defaults to distribution)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("acquire-by-hash", false, "provide index files by hash")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_switch.go
+5 -5
View File
@@ -5,7 +5,6 @@ import (
"strings"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/utils"
"github.com/smira/commander"
"github.com/smira/flag"
)
@@ -64,10 +63,6 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
}
for i, component := range components {
if !utils.StrSliceHasItem(publishedComponents, component) {
return fmt.Errorf("unable to switch: component %s is not in published repository", component)
}
snapshot, err = collectionFactory.SnapshotCollection().ByName(names[i])
if err != nil {
return fmt.Errorf("unable to switch: %s", err)
@@ -100,6 +95,10 @@ func aptlyPublishSwitch(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
@@ -161,6 +160,7 @@ This command would switch published repository (with one component) named ppa/wh
cmd.Flag.String("component", "", "component names to update (for multi-component publishing, separate components with commas)")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/publish_update.go
+5
View File
@@ -64,6 +64,10 @@ func aptlyPublishUpdate(cmd *commander.Command, args []string) error {
published.SkipBz2 = context.Flags().Lookup("skip-bz2").Value.Get().(bool)
}
if context.Flags().IsSet("multi-dist") {
published.MultiDist = context.Flags().Lookup("multi-dist").Value.Get().(bool)
}
err = published.Publish(context.PackagePool(), context, collectionFactory, signer, context.Progress(), forceOverwrite)
if err != nil {
return fmt.Errorf("unable to publish: %s", err)
@@ -119,6 +123,7 @@ Example:
cmd.Flag.Bool("skip-bz2", false, "don't generate bzipped indexes")
cmd.Flag.Bool("force-overwrite", false, "overwrite files in package pool in case of mismatch")
cmd.Flag.Bool("skip-cleanup", false, "don't remove unreferenced files in prefix/component")
cmd.Flag.Bool("multi-dist", false, "enable multiple packages with the same filename in different distributions")
return cmd
}
cmd/repo_include.go
+4 -1
View File
@@ -29,7 +29,10 @@ func aptlyRepoInclude(cmd *commander.Command, args []string) error {
forceReplace := context.Flags().Lookup("force-replace").Value.Get().(bool)
acceptUnsigned := context.Flags().Lookup("accept-unsigned").Value.Get().(bool)
ignoreSignatures := context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
ignoreSignatures := context.Config().GpgDisableVerify
if context.Flags().IsSet("ignore-signatures") {
ignoreSignatures = context.Flags().Lookup("ignore-signatures").Value.Get().(bool)
}
noRemoveFiles := context.Flags().Lookup("no-remove-files").Value.Get().(bool)
repoTemplateString := context.Flags().Lookup("repo").Value.Get().(string)
collectionFactory := context.NewCollectionFactory()
cmd/repo_list.go
+2 -2
View File
@@ -24,7 +24,7 @@ func aptlyRepoList(cmd *commander.Command, args []string) error {
return aptlyRepoListTxt(cmd, args)
}
func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
func aptlyRepoListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -71,7 +71,7 @@ func aptlyRepoListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyRepoListJSON(cmd *commander.Command, args []string) error {
func aptlyRepoListJSON(_ *commander.Command, _ []string) error {
var err error
repos := make([]*deb.LocalRepo, context.NewCollectionFactory().LocalRepoCollection().Len())
cmd/repo_show.go
+2 -2
View File
@@ -25,7 +25,7 @@ func aptlyRepoShow(cmd *commander.Command, args []string) error {
return aptlyRepoShowTxt(cmd, args)
}
func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
func aptlyRepoShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
@@ -58,7 +58,7 @@ func aptlyRepoShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlyRepoShowJSON(cmd *commander.Command, args []string) error {
func aptlyRepoShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/serve.go
+1 -1
View File
@@ -29,7 +29,7 @@ func aptlyServe(cmd *commander.Command, args []string) error {
// anything else must fail.
// E.g.: Running the service under a different user may lead to a rootDir
// that exists but is not usable due to access permissions.
err = utils.DirIsAccessible(context.Config().RootDir)
err = utils.DirIsAccessible(context.Config().GetRootDir())
if err != nil {
return err
}
cmd/snapshot_list.go
+2 -2
View File
@@ -23,7 +23,7 @@ func aptlySnapshotList(cmd *commander.Command, args []string) error {
return aptlySnapshotListTxt(cmd, args)
}
func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
func aptlySnapshotListTxt(cmd *commander.Command, _ []string) error {
var err error
raw := cmd.Flag.Lookup("raw").Value.Get().(bool)
@@ -59,7 +59,7 @@ func aptlySnapshotListTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlySnapshotListJSON(cmd *commander.Command, args []string) error {
func aptlySnapshotListJSON(cmd *commander.Command, _ []string) error {
var err error
sortMethodString := cmd.Flag.Lookup("sort").Value.Get().(string)
cmd/snapshot_show.go
+2 -2
View File
@@ -25,7 +25,7 @@ func aptlySnapshotShow(cmd *commander.Command, args []string) error {
return aptlySnapshotShowTxt(cmd, args)
}
func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
func aptlySnapshotShowTxt(_ *commander.Command, args []string) error {
var err error
name := args[0]
collectionFactory := context.NewCollectionFactory()
@@ -85,7 +85,7 @@ func aptlySnapshotShowTxt(cmd *commander.Command, args []string) error {
return err
}
func aptlySnapshotShowJSON(cmd *commander.Command, args []string) error {
func aptlySnapshotShowJSON(_ *commander.Command, args []string) error {
var err error
name := args[0]
cmd/task_run.go
+3 -4
View File
@@ -62,11 +62,10 @@ func aptlyTaskRun(cmd *commander.Command, args []string) error {
text, _ = reader.ReadString('\n')
if text == "\n" {
break
} else {
text = strings.TrimSpace(text) + ","
parsedArgs, _ := shellwords.Parse(text)
cmdArgs = append(cmdArgs, parsedArgs...)
}
text = strings.TrimSpace(text) + ","
parsedArgs, _ := shellwords.Parse(text)
cmdArgs = append(cmdArgs, parsedArgs...)
}
if len(cmdArgs) == 0 {
codecov.yml
+1 -1
View File
@@ -3,5 +3,5 @@ coverage:
project:
default:
target: auto
threshold: 0%
threshold: 2%
if_ci_failed: error
completion.d/_aptly
+1
View File
@@ -457,6 +457,7 @@ local keyring="*-keyring=[gpg keyring to use when verifying Release file (could
"-distribution=[distribution name to publish]:distribution:($dists)"
"-label=[label to publish]:label: "
"-suite=[suite to publish]:suite: "
"-codename=[codename to publish]:codename: "
"-notautomatic=[set value for NotAutomatic field]:notautomatic: "
"-origin=[origin name to publish]:origin: "
${components_options[@]}
completion.d/aptly
+1 -3
View File
@@ -1,5 +1,3 @@
#!/bin/bash
# (The MIT License)
#
# Copyright (c) 2014 Andrey Smirnov
@@ -503,7 +501,7 @@ _aptly()
"snapshot"|"repo")
if [[ $numargs -eq 0 ]]; then
if [[ "$cur" == -* ]]; then
COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing" -- ${cur}))
COMPREPLY=($(compgen -W "-acquire-by-hash -batch -butautomaticupgrades= -component= -distribution= -force-overwrite -gpg-key= -keyring= -label= -suite= -codename= -notautomatic= -origin= -passphrase= -passphrase-file= -secret-keyring= -skip-contents -skip-bz2 -skip-signing -multi-dist" -- ${cur}))
else
if [[ "$subcmd" == "snapshot" ]]; then
COMPREPLY=($(compgen -W "$(__aptly_snapshot_list)" -- ${cur}))
console/progress.go
+71 -14
View File
@@ -7,6 +7,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/cheggaaa/pb"
"github.com/rs/zerolog/log"
"github.com/wsxiaoys/terminal/color"
)
@@ -34,6 +35,7 @@ type Progress struct {
queue chan printTask
bar *pb.ProgressBar
barShown bool
worker ProgressWorker
}
// Check interface
@@ -42,16 +44,19 @@ var (
)
// NewProgress creates new progress instance
func NewProgress() *Progress {
return &Progress{
func NewProgress(structuredLogging bool) *Progress {
p := &Progress{
stopped: make(chan bool),
queue: make(chan printTask, 100),
}
p.worker = progressWorkerFactroy(structuredLogging, p)
return p
}
// Start makes progress start its work
func (p *Progress) Start() {
go p.worker()
go p.worker.run()
}
// Shutdown shuts down progress display
@@ -69,7 +74,7 @@ func (p *Progress) Flush() {
}
// InitBar starts progressbar for count bytes or count items
func (p *Progress) InitBar(count int64, isBytes bool, barType aptly.BarType) {
func (p *Progress) InitBar(count int64, isBytes bool, _ aptly.BarType) {
if p.bar != nil {
panic("bar already initialized")
}
@@ -173,42 +178,94 @@ func (p *Progress) ColoredPrintf(msg string, a ...interface{}) {
}
}
func (p *Progress) worker() {
type ProgressWorker interface {
run()
}
func progressWorkerFactroy(structuredLogging bool, progress *Progress) ProgressWorker {
if structuredLogging {
worker := loggerProgressWorker{progress: progress}
return &worker
}
worker := standardProgressWorker{progress: progress}
return &worker
}
type standardProgressWorker struct {
progress *Progress
}
func (w *standardProgressWorker) run() {
hasBar := false
for {
task := <-p.queue
task := <-w.progress.queue
switch task.code {
case codeBarEnabled:
hasBar = true
case codeBarDisabled:
hasBar = false
case codePrint:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
fmt.Print(task.message)
case codePrintStdErr:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
fmt.Fprint(os.Stderr, task.message)
case codeProgress:
if hasBar {
fmt.Print("\r" + task.message)
p.barShown = true
w.progress.barShown = true
}
case codeHideProgress:
if p.barShown {
if w.progress.barShown {
fmt.Print("\r\033[2K")
p.barShown = false
w.progress.barShown = false
}
case codeFlush:
task.reply <- true
case codeStop:
p.stopped <- true
w.progress.stopped <- true
return
}
}
}
type loggerProgressWorker struct {
progress *Progress
}
func (w *loggerProgressWorker) run() {
hasBar := false
for {
task := <-w.progress.queue
switch task.code {
case codeBarEnabled:
hasBar = true
case codeBarDisabled:
hasBar = false
case codePrint, codePrintStdErr:
log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
case codeProgress:
if hasBar {
log.Info().Msg(strings.TrimSuffix(task.message, "\n"))
w.progress.barShown = true
}
case codeHideProgress:
if w.progress.barShown {
w.progress.barShown = false
}
case codeFlush:
task.reply <- true
case codeStop:
w.progress.stopped <- true
return
}
}
console/progress_test.go
+24
View File
@@ -0,0 +1,24 @@
package console
import (
"fmt"
"testing"
. "gopkg.in/check.v1"
)
func Test(t *testing.T) {
TestingT(t)
}
type ProgressSuite struct {}
var _ = Suite(&ProgressSuite{})
func (s *ProgressSuite) TestNewProgress(c *C) {
p := NewProgress(false)
c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &standardProgressWorker{}))
p = NewProgress(true)
c.Check(fmt.Sprintf("%T", p.worker), Equals, fmt.Sprintf("%T", &loggerProgressWorker{}))
}
context/context.go
+58 -16
View File
@@ -3,6 +3,7 @@ package context
import (
gocontext "context"
"errors"
"fmt"
"math/rand"
"os"
@@ -18,6 +19,7 @@ import (
"github.com/aptly-dev/aptly/azure"
"github.com/aptly-dev/aptly/console"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/etcddb"
"github.com/aptly-dev/aptly/database/goleveldb"
"github.com/aptly-dev/aptly/deb"
"github.com/aptly-dev/aptly/files"
@@ -48,6 +50,7 @@ type AptlyContext struct {
publishedStorages map[string]aptly.PublishedStorage
dependencyOptions int
architecturesList []string
structuredLogging bool
// Debug features
fileCPUProfile *os.File
fileMemProfile *os.File
@@ -93,13 +96,15 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
Fatal(err)
}
} else {
configLocations := []string{
filepath.Join(os.Getenv("HOME"), ".aptly.conf"),
"/etc/aptly.conf",
}
homeLocation := filepath.Join(os.Getenv("HOME"), ".aptly.conf")
configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"}
for _, configLocation := range configLocations {
// FIXME: check if exists, check if readable
err = utils.LoadConfig(configLocation, &utils.Config)
if os.IsPermission(err) || os.IsNotExist(err) {
continue
}
if err == nil {
break
}
@@ -109,7 +114,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure {
}
if err != nil {
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", configLocations[0])
fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation)
// as this is fresh aptly installation, we don't need to support legacy pool locations
utils.Config.SkipLegacyPool = true
@@ -195,7 +200,7 @@ func (context *AptlyContext) Progress() aptly.Progress {
func (context *AptlyContext) _progress() aptly.Progress {
if context.progress == nil {
context.progress = console.NewProgress()
context.progress = console.NewProgress(context.structuredLogging)
context.progress.Start()
}
@@ -272,7 +277,7 @@ func (context *AptlyContext) DBPath() string {
// DBPath builds path to database
func (context *AptlyContext) dbPath() string {
return filepath.Join(context.config().RootDir, "db")
return filepath.Join(context.config().GetRootDir(), "db")
}
// Database opens and returns current instance of database
@@ -286,8 +291,18 @@ func (context *AptlyContext) Database() (database.Storage, error) {
func (context *AptlyContext) _database() (database.Storage, error) {
if context.database == nil {
var err error
context.database, err = goleveldb.NewDB(context.dbPath())
switch context.config().DatabaseBackend.Type {
case "leveldb":
if len(context.config().DatabaseBackend.DbPath) == 0 {
return nil, errors.New("leveldb databaseBackend config invalid")
}
dbPath := filepath.Join(context.config().RootDir, context.config().DatabaseBackend.DbPath)
context.database, err = goleveldb.NewDB(dbPath)
case "etcd":
context.database, err = etcddb.NewDB(context.config().DatabaseBackend.URL)
default:
context.database, err = goleveldb.NewDB(context.dbPath())
}
if err != nil {
return nil, fmt.Errorf("can't instantiate database: %s", err)
}
@@ -360,7 +375,26 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool {
defer context.Unlock()
if context.packagePool == nil {
context.packagePool = files.NewPackagePool(context.config().RootDir, !context.config().SkipLegacyPool)
storageConfig := context.config().PackagePoolStorage
if storageConfig.Azure != nil {
var err error
context.packagePool, err = azure.NewPackagePool(
storageConfig.Azure.AccountName,
storageConfig.Azure.AccountKey,
storageConfig.Azure.Container,
storageConfig.Azure.Prefix,
storageConfig.Azure.Endpoint)
if err != nil {
Fatal(err)
}
} else {
poolRoot := context.config().PackagePoolStorage.Local.Path
if poolRoot == "" {
poolRoot = filepath.Join(context.config().RootDir, "pool")
}
context.packagePool = files.NewPackagePool(poolRoot, !context.config().SkipLegacyPool)
}
}
return context.packagePool
@@ -374,7 +408,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
publishedStorage, ok := context.publishedStorages[name]
if !ok {
if name == "" {
publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().RootDir, "public"), "hardlink", "")
publishedStorage = files.NewPublishedStorage(filepath.Join(context.config().GetRootDir(), "public"), "hardlink", "")
} else if strings.HasPrefix(name, "filesystem:") {
params, ok := context.config().FileSystemPublishRoots[name[11:]]
if !ok {
@@ -393,7 +427,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
params.AccessKeyID, params.SecretAccessKey, params.SessionToken,
params.Region, params.Endpoint, params.Bucket, params.ACL, params.Prefix, params.StorageClass,
params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel,
params.ForceSigV2, params.Debug)
params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug)
if err != nil {
Fatal(err)
}
@@ -432,7 +466,7 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto
// UploadPath builds path to upload storage
func (context *AptlyContext) UploadPath() string {
return filepath.Join(context.Config().RootDir, "upload")
return filepath.Join(context.Config().GetRootDir(), "upload")
}
func (context *AptlyContext) pgpProvider() string {
@@ -456,7 +490,7 @@ func (context *AptlyContext) pgpProvider() string {
return provider
}
func (context *AptlyContext) getGPGFinder(provider string) pgp.GPGFinder {
func (context *AptlyContext) getGPGFinder() pgp.GPGFinder {
switch context.pgpProvider() {
case "gpg1":
return pgp.GPG1Finder()
@@ -479,7 +513,7 @@ func (context *AptlyContext) GetSigner() pgp.Signer {
return &pgp.GoSigner{}
}
return pgp.NewGpgSigner(context.getGPGFinder(provider))
return pgp.NewGpgSigner(context.getGPGFinder())
}
// GetVerifier returns Verifier with respect to provider
@@ -492,7 +526,7 @@ func (context *AptlyContext) GetVerifier() pgp.Verifier {
return &pgp.GoVerifier{}
}
return pgp.NewGpgVerifier(context.getGPGFinder(provider))
return pgp.NewGpgVerifier(context.getGPGFinder())
}
// UpdateFlags sets internal copy of flags in the context
@@ -540,6 +574,11 @@ func (context *AptlyContext) GoContextHandleSignals() {
}()
}
// StructuredLogging allows to set the structuredLogging flag
func (context *AptlyContext) StructuredLogging(structuredLogging bool) {
context.structuredLogging = structuredLogging
}
// Shutdown shuts context down
func (context *AptlyContext) Shutdown() {
context.Lock()
@@ -561,6 +600,9 @@ func (context *AptlyContext) Shutdown() {
context.fileMemProfile = nil
}
}
if context.taskList != nil {
context.taskList.Stop()
}
if context.database != nil {
context.database.Close()
context.database = nil
database/etcddb/batch.go
+53
View File
@@ -0,0 +1,53 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
type EtcDBatch struct {
s *EtcDStorage
ops []clientv3.Op
}
type WriteOptions struct {
NoWriteMerge bool
Sync bool
}
func (b *EtcDBatch) Put(key []byte, value []byte) (err error) {
b.ops = append(b.ops, clientv3.OpPut(string(key), string(value)))
return
}
func (b *EtcDBatch) Delete(key []byte) (err error) {
b.ops = append(b.ops, clientv3.OpDelete(string(key)))
return
}
func (b *EtcDBatch) Write() (err error) {
kv := clientv3.NewKV(b.s.db)
batchSize := 128
for i := 0; i < len(b.ops); i += batchSize {
txn := kv.Txn(Ctx)
end := i + batchSize
if end > len(b.ops) {
end = len(b.ops)
}
batch := b.ops[i:end]
txn.Then(batch...)
_, err = txn.Commit()
if err != nil {
panic(err)
}
}
return
}
// batch should implement database.Batch
var (
_ database.Batch = &EtcDBatch{}
)
database/etcddb/database.go
+32
View File
@@ -0,0 +1,32 @@
package etcddb
import (
"context"
"time"
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
var Ctx = context.TODO()
func internalOpen(url string) (cli *clientv3.Client, err error) {
cfg := clientv3.Config{
Endpoints: []string{url},
DialTimeout: 30 * time.Second,
MaxCallSendMsgSize: 2147483647, // (2048 * 1024 * 1024) - 1
MaxCallRecvMsgSize: 2147483647,
DialKeepAliveTimeout: 7200 * time.Second,
}
cli, err = clientv3.New(cfg)
return
}
func NewDB(url string) (database.Storage, error) {
cli, err := internalOpen(url)
if err != nil {
return nil, err
}
return &EtcDStorage{url, cli, ""}, nil
}
database/etcddb/database_test.go
+158
View File
@@ -0,0 +1,158 @@
package etcddb_test
import (
"testing"
"github.com/aptly-dev/aptly/database"
"github.com/aptly-dev/aptly/database/etcddb"
. "gopkg.in/check.v1"
)
// Launch gocheck tests
func Test(t *testing.T) {
TestingT(t)
}
type EtcDDBSuite struct {
url string
db database.Storage
}
var _ = Suite(&EtcDDBSuite{})
func (s *EtcDDBSuite) SetUpTest(c *C) {
var err error
s.db, err = etcddb.NewDB("127.0.0.1:2379")
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestSetUpTest(c *C) {
var err error
s.db, err = etcddb.NewDB("127.0.0.1:2379")
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestGetPut(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
var err error
err = s.db.Put(key, value)
c.Assert(err, IsNil)
result, err := s.db.Get(key)
c.Assert(err, IsNil)
c.Assert(result, DeepEquals, value)
}
func (s *EtcDDBSuite) TestDelete(c *C) {
var (
key = []byte("key")
value = []byte("value")
)
err := s.db.Put(key, value)
c.Assert(err, IsNil)
_, err = s.db.Get(key)
c.Assert(err, IsNil)
err = s.db.Delete(key)
c.Assert(err, IsNil)
}
func (s *EtcDDBSuite) TestByPrefix(c *C) {
//c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{})
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
s.db.Put([]byte{0x80, 0x03}, []byte{0x03})
s.db.Put([]byte{0x80, 0x02}, []byte{0x02})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
s.db.Put([]byte{0x90, 0x01}, []byte{0x04})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
s.db.Put([]byte{0x00, 0x01}, []byte{0x05})
c.Check(s.db.FetchByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(s.db.KeysByPrefix([]byte{0x80}), DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
keys := [][]byte{}
values := [][]byte{}
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
keys = append(keys, append([]byte(nil), k...))
values = append(values, append([]byte(nil), v...))
return nil
}), IsNil)
c.Check(values, DeepEquals, [][]byte{{0x01}, {0x02}, {0x03}})
c.Check(keys, DeepEquals, [][]byte{{0x80, 0x01}, {0x80, 0x02}, {0x80, 0x03}})
c.Check(s.db.ProcessByPrefix([]byte{0x80}, func(k, v []byte) error {
return database.ErrNotFound
}), Equals, database.ErrNotFound)
c.Check(s.db.ProcessByPrefix([]byte{0xa0}, func(k, v []byte) error {
return database.ErrNotFound
}), IsNil)
c.Check(s.db.FetchByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
c.Check(s.db.KeysByPrefix([]byte{0xa0}), DeepEquals, [][]byte{})
}
func (s *EtcDDBSuite) TestHasPrefix(c *C) {
//c.Check(s.db.HasPrefix([]byte(nil)), Equals, false)
//c.Check(s.db.HasPrefix([]byte{0x80}), Equals, false)
s.db.Put([]byte{0x80, 0x01}, []byte{0x01})
c.Check(s.db.HasPrefix([]byte(nil)), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x80}), Equals, true)
c.Check(s.db.HasPrefix([]byte{0x79}), Equals, false)
}
func (s *EtcDDBSuite) TestTransactionCommit(c *C) {
var (
key = []byte("key")
key2 = []byte("key2")
value = []byte("value")
value2 = []byte("value2")
)
transaction, err := s.db.OpenTransaction()
err = s.db.Put(key, value)
c.Assert(err, IsNil)
c.Assert(err, IsNil)
transaction.Put(key2, value2)
v, err := s.db.Get(key)
c.Check(v, DeepEquals, value)
err = transaction.Delete(key)
c.Assert(err, IsNil)
_, err = transaction.Get(key2)
c.Assert(err, IsNil)
v2, err := transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Assert(err, IsNil)
err = transaction.Commit()
c.Check(err, IsNil)
v2, err = transaction.Get(key2)
c.Check(err, IsNil)
c.Check(v2, DeepEquals, value2)
_, err = transaction.Get(key)
c.Assert(err, NotNil)
}
database/etcddb/storage.go
+202
View File
@@ -0,0 +1,202 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
"github.com/pborman/uuid"
clientv3 "go.etcd.io/etcd/client/v3"
"fmt"
)
type EtcDStorage struct {
url string
db *clientv3.Client
tmpPrefix string // prefix for temporary DBs
}
// CreateTemporary creates new DB of the same type in temp dir
func (s *EtcDStorage) CreateTemporary() (database.Storage, error) {
tmp := uuid.NewRandom().String()
return &EtcDStorage{
url: s.url,
db: s.db,
tmpPrefix: tmp,
}, nil
}
func (s *EtcDStorage) applyPrefix(key []byte) []byte {
if len(s.tmpPrefix) != 0 {
return append([]byte(s.tmpPrefix+"/"), key...)
}
return key
}
// Get key value from etcd
func (s *EtcDStorage) Get(key []byte) (value []byte, err error) {
realKey := s.applyPrefix(key)
getResp, err := s.db.Get(Ctx, string(realKey))
if err != nil {
return
}
for _, kv := range getResp.Kvs {
value = kv.Value
break
}
if len(value) == 0 {
err = database.ErrNotFound
return
}
return
}
// Put saves key to etcd, if key has the same value in DB already, it is not saved
func (s *EtcDStorage) Put(key []byte, value []byte) (err error) {
realKey := s.applyPrefix(key)
_, err = s.db.Put(Ctx, string(realKey), string(value))
if err != nil {
return
}
return
}
// Delete removes key from etcd
func (s *EtcDStorage) Delete(key []byte) (err error) {
realKey := s.applyPrefix(key)
_, err = s.db.Delete(Ctx, string(realKey))
if err != nil {
return
}
return
}
// KeysByPrefix returns all keys that start with prefix
func (s *EtcDStorage) KeysByPrefix(prefix []byte) [][]byte {
realPrefix := s.applyPrefix(prefix)
result := make([][]byte, 0, 20)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return nil
}
for _, ev := range getResp.Kvs {
key := ev.Key
keyc := make([]byte, len(key))
copy(keyc, key)
result = append(result, key)
}
return result
}
// FetchByPrefix returns all values with keys that start with prefix
func (s *EtcDStorage) FetchByPrefix(prefix []byte) [][]byte {
realPrefix := s.applyPrefix(prefix)
result := make([][]byte, 0, 20)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return nil
}
for _, kv := range getResp.Kvs {
valc := make([]byte, len(kv.Value))
copy(valc, kv.Value)
result = append(result, kv.Value)
}
return result
}
// HasPrefix checks whether it can find any key with given prefix and returns true if one exists
func (s *EtcDStorage) HasPrefix(prefix []byte) bool {
realPrefix := s.applyPrefix(prefix)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return false
}
return getResp.Count > 0
}
// ProcessByPrefix iterates through all entries where key starts with prefix and calls
// StorageProcessor on key value pair
func (s *EtcDStorage) ProcessByPrefix(prefix []byte, proc database.StorageProcessor) error {
realPrefix := s.applyPrefix(prefix)
getResp, err := s.db.Get(Ctx, string(realPrefix), clientv3.WithPrefix())
if err != nil {
return err
}
for _, kv := range getResp.Kvs {
err := proc(kv.Key, kv.Value)
if err != nil {
return err
}
}
return nil
}
// Close finishes etcd connect
func (s *EtcDStorage) Close() error {
// do not close temporary db
if len(s.tmpPrefix) != 0 {
return nil
}
if s.db == nil {
return nil
}
err := s.db.Close()
s.db = nil
return err
}
// Reopen tries to open (re-open) the database
func (s *EtcDStorage) Open() error {
if s.db != nil {
return nil
}
var err error
s.db, err = internalOpen(s.url)
return err
}
// CreateBatch creates a Batch object
func (s *EtcDStorage) CreateBatch() database.Batch {
if s.db == nil {
return nil
}
return &EtcDBatch{
s: s,
}
}
// OpenTransaction creates new transaction.
func (s *EtcDStorage) OpenTransaction() (database.Transaction, error) {
tmpdb, err := s.CreateTemporary()
if err != nil {
return nil, err
}
return &transaction{s: s, tmpdb: tmpdb}, nil
}
// CompactDB does nothing for etcd
func (s *EtcDStorage) CompactDB() error {
return nil
}
// Drop removes only temporary DBs with etcd (i.e. remove all prefixed keys)
func (s *EtcDStorage) Drop() error {
if len(s.tmpPrefix) != 0 {
getResp, err := s.db.Get(Ctx, s.tmpPrefix, clientv3.WithPrefix())
if err != nil {
return nil
}
for _, kv := range getResp.Kvs {
_, err = s.db.Delete(Ctx, string(kv.Key))
if err != nil {
return fmt.Errorf("cannot delete tempdb entry: %s", kv.Key)
}
}
}
return nil
}
// Check interface
var (
_ database.Storage = &EtcDStorage{}
)
database/etcddb/transaction.go
+75
View File
@@ -0,0 +1,75 @@
package etcddb
import (
"github.com/aptly-dev/aptly/database"
clientv3 "go.etcd.io/etcd/client/v3"
)
type transaction struct {
s *EtcDStorage
tmpdb database.Storage
ops []clientv3.Op
}
// Get implements database.Reader interface.
func (t *transaction) Get(key []byte) (value []byte, err error) {
value, err = t.tmpdb.Get(key)
// if not found, search main db
if err != nil {
value, err = t.s.Get(key)
}
return
}
// Put implements database.Writer interface.
func (t *transaction) Put(key, value []byte) (err error) {
err = t.tmpdb.Put(key, value)
if err != nil {
return
}
t.ops = append(t.ops, clientv3.OpPut(string(key), string(value)))
return
}
// Delete implements database.Writer interface.
func (t *transaction) Delete(key []byte) (err error) {
err = t.tmpdb.Delete(key)
if err != nil {
return
}
t.ops = append(t.ops, clientv3.OpDelete(string(key)))
return
}
func (t *transaction) Commit() (err error) {
kv := clientv3.NewKV(t.s.db)
batchSize := 128
for i := 0; i < len(t.ops); i += batchSize {
txn := kv.Txn(Ctx)
end := i + batchSize
if end > len(t.ops) {
end = len(t.ops)
}
batch := t.ops[i:end]
txn.Then(batch...)
_, err = txn.Commit()
if err != nil {
panic(err)
}
}
t.ops = []clientv3.Op{}
return
}
// Discard is safe to call after Commit(), it would be no-op
func (t *transaction) Discard() {
t.ops = []clientv3.Op{}
t.tmpdb.Drop()
return
}
// transaction should implement database.Transaction
var _ database.Transaction = &transaction{}
database/goleveldb/storage.go
+1 -2
View File
@@ -3,7 +3,6 @@ package goleveldb
import (
"bytes"
"errors"
"io/ioutil"
"os"
"github.com/syndtr/goleveldb/leveldb"
@@ -19,7 +18,7 @@ type storage struct {
// CreateTemporary creates new DB of the same type in temp dir
func (s *storage) CreateTemporary() (database.Storage, error) {
tempdir, err := ioutil.TempDir("", "aptly")
tempdir, err := os.MkdirTemp("", "aptly")
if err != nil {
return nil, err
}
deb/changes.go
+10 -8
View File
@@ -4,16 +4,17 @@ import (
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"text/template"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/saracen/walker"
)
// Changes is a result of .changes file parsing
@@ -39,7 +40,7 @@ func NewChanges(path string) (*Changes, error) {
ChangesName: filepath.Base(path),
}
c.TempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
c.TempDir, err = os.MkdirTemp(os.TempDir(), "aptly")
if err != nil {
return nil, err
}
@@ -221,12 +222,12 @@ func (c *Changes) GetField(field string) string {
}
// MatchesDependency implements PackageLike interface
func (c *Changes) MatchesDependency(d Dependency) bool {
func (c *Changes) MatchesDependency(_ Dependency) bool {
return false
}
// MatchesArchitecture implements PackageLike interface
func (c *Changes) MatchesArchitecture(arch string) bool {
func (c *Changes) MatchesArchitecture(_ string) bool {
return false
}
@@ -248,6 +249,8 @@ func (c *Changes) GetArchitecture() string {
// CollectChangesFiles walks filesystem collecting all .changes files
func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) {
changesFilesLock := &sync.Mutex{}
for _, location := range locations {
info, err2 := os.Stat(location)
if err2 != nil {
@@ -256,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha
continue
}
if info.IsDir() {
err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
if err3 != nil {
return err3
}
err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
if info.IsDir() {
return nil
}
if strings.HasSuffix(info.Name(), ".changes") {
changesFilesLock.Lock()
defer changesFilesLock.Unlock()
changesFiles = append(changesFiles, path)
}
deb/changes_test.go
+1 -1
View File
@@ -45,7 +45,7 @@ func (s *ChangesSuite) SetUpTest(c *C) {
s.checksumStorage = files.NewMockChecksumStorage()
s.packagePool = files.NewPackagePool(s.Dir, false)
s.progress = console.NewProgress()
s.progress = console.NewProgress(false)
s.progress.Start()
}
deb/contents.go
+1 -1
View File
@@ -64,7 +64,7 @@ func (index *ContentsIndex) WriteTo(w io.Writer) (int64, error) {
currentPkgs [][]byte
)
err = index.db.ProcessByPrefix(index.prefix, func(key []byte, value []byte) error {
err = index.db.ProcessByPrefix(index.prefix, func(key []byte, _ []byte) error {
// cut prefix
key = key[prefixLen:]
deb/format.go
+1 -1
View File
@@ -136,7 +136,7 @@ func isMultilineField(field string, isRelease bool) bool {
// Write single field from Stanza to writer.
//
//nolint: interfacer
// nolint: interfacer
func writeField(w *bufio.Writer, field, value string, isRelease bool) (err error) {
if !isMultilineField(field, isRelease) {
_, err = w.WriteString(field + ": " + value + "\n")
deb/graph.go
+2 -2
View File
@@ -119,8 +119,8 @@ func BuildGraph(collectionFactory *CollectionFactory, layout string) (gographviz
"shape": "Mrecord",
"style": "filled",
"fillcolor": "darkolivegreen1",
"label": fmt.Sprintf("%sPublished %s/%s|comp: %s|arch: %s%s", labelStart,
repo.Prefix, repo.Distribution, strings.Join(repo.Components(), " "),
"label": fmt.Sprintf("%sPublished %s|comp: %s|arch: %s%s", labelStart,
repo.GetPath(), strings.Join(repo.Components(), " "),
strings.Join(repo.Architectures, ", "), labelEnd),
})
deb/import.go
+13 -13
View File
@@ -5,14 +5,19 @@ import (
"path/filepath"
"sort"
"strings"
"sync"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/pgp"
"github.com/aptly-dev/aptly/utils"
"github.com/saracen/walker"
)
// CollectPackageFiles walks filesystem collecting all candidates for package files
func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) {
packageFilesLock := &sync.Mutex{}
otherFilesLock := &sync.Mutex{}
for _, location := range locations {
info, err2 := os.Stat(location)
if err2 != nil {
@@ -21,18 +26,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac
continue
}
if info.IsDir() {
err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error {
if err3 != nil {
return err3
}
err2 = walker.Walk(location, func(path string, info os.FileInfo) error {
if info.IsDir() {
return nil
}
if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") ||
strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") {
packageFilesLock.Lock()
defer packageFilesLock.Unlock()
packageFiles = append(packageFiles, path)
} else if strings.HasSuffix(info.Name(), ".buildinfo") {
otherFilesLock.Lock()
defer otherFilesLock.Unlock()
otherFiles = append(otherFiles, path)
}
@@ -71,13 +77,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
list.PrepareIndex()
}
transaction, err := collection.db.OpenTransaction()
if err != nil {
return nil, nil, err
}
defer transaction.Discard()
checksumStorage := checksumStorageProvider(transaction)
checksumStorage := checksumStorageProvider(collection.db)
for _, file := range packageFiles {
var (
@@ -201,7 +201,7 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
continue
}
err = collection.UpdateInTransaction(p, transaction)
err = collection.Update(p)
if err != nil {
reporter.Warning("Unable to save package %s: %s", p, err)
failedFiles = append(failedFiles, file)
@@ -227,6 +227,6 @@ func ImportPackageFiles(list *PackageList, packageFiles []string, forceReplace b
processedFiles = append(processedFiles, candidateProcessedFiles...)
}
err = transaction.Commit()
err = nil // reset error as only failed files are reported
return
}
deb/index_files.go
+12 -7
View File
@@ -143,19 +143,20 @@ func (file *indexFile) Finalize(signer pgp.Signer) error {
}
if signer != nil {
gpgExt := ".gpg"
if file.detachedSign {
err = signer.DetachedSign(file.tempFilename, file.tempFilename+".gpg")
err = signer.DetachedSign(file.tempFilename, file.tempFilename+gpgExt)
if err != nil {
return fmt.Errorf("unable to detached sign file: %s", err)
}
if file.parent.suffix != "" {
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg")] =
filepath.Join(file.parent.basePath, file.relativePath+".gpg")
file.parent.renameMap[filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt)] =
filepath.Join(file.parent.basePath, file.relativePath+gpgExt)
}
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+".gpg"),
file.tempFilename+".gpg")
err = file.parent.publishedStorage.PutFile(filepath.Join(file.parent.basePath, file.relativePath+file.parent.suffix+gpgExt),
file.tempFilename+gpgExt)
if err != nil {
return fmt.Errorf("unable to publish file: %s", err)
}
@@ -248,7 +249,7 @@ func newIndexFiles(publishedStorage aptly.PublishedStorage, basePath, tempDir, s
}
}
func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bool) *indexFile {
func (files *indexFiles) PackageIndex(component, arch string, udeb bool, installer bool, distribution string) *indexFile {
if arch == ArchitectureSource {
udeb = false
}
@@ -263,7 +264,11 @@ func (files *indexFiles) PackageIndex(component, arch string, udeb, installer bo
if udeb {
relativePath = filepath.Join(component, "debian-installer", fmt.Sprintf("binary-%s", arch), "Packages")
} else if installer {
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
if distribution == aptly.DistributionFocal {
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "legacy-images", "SHA256SUMS")
} else {
relativePath = filepath.Join(component, fmt.Sprintf("installer-%s", arch), "current", "images", "SHA256SUMS")
}
} else {
relativePath = filepath.Join(component, fmt.Sprintf("binary-%s", arch), "Packages")
}
deb/list.go
+17 -11
View File
@@ -145,7 +145,7 @@ func (l *PackageList) Add(p *Package) error {
l.packages[key] = p
if l.indexed {
for _, provides := range p.Provides {
for _, provides := range p.ProvidedPackages() {
l.providesIndex[provides] = append(l.providesIndex[provides], p)
}
@@ -215,7 +215,7 @@ func (l *PackageList) Append(pl *PackageList) error {
func (l *PackageList) Remove(p *Package) {
delete(l.packages, l.keyFunc(p))
if l.indexed {
for _, provides := range p.Provides {
for _, provides := range p.ProvidedPackages() {
for i, pkg := range l.providesIndex[provides] {
if pkg.Equals(p) {
// remove l.ProvidesIndex[provides][i] w/o preserving order
@@ -351,7 +351,7 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
cache[hash] = satisfied
}
if !satisfied && !ok {
if !satisfied {
variantsMissing = append(variantsMissing, dep)
}
@@ -366,6 +366,8 @@ func (l *PackageList) VerifyDependencies(options int, architectures []string, so
}
}
missing = depSliceDeduplicate(missing)
if progress != nil {
progress.ShutdownBar()
}
@@ -417,7 +419,7 @@ func (l *PackageList) PrepareIndex() {
l.packagesIndex[i] = p
i++
for _, provides := range p.Provides {
for _, provides := range p.ProvidedPackages() {
l.providesIndex[provides] = append(l.providesIndex[provides], p)
}
}
@@ -470,21 +472,25 @@ func (l *PackageList) Search(dep Dependency, allMatches bool) (searchResults []*
searchResults = append(searchResults, p)
if !allMatches {
break
return
}
}
i++
}
if dep.Relation == VersionDontCare {
for _, p := range l.providesIndex[dep.Pkg] {
if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
providers, ok := l.providesIndex[dep.Pkg]
if !ok {
return
}
for _, p := range providers {
if dep.Architecture == "" || p.MatchesArchitecture(dep.Architecture) {
if p.MatchesDependency(dep) {
searchResults = append(searchResults, p)
}
if !allMatches {
break
}
if !allMatches {
return
}
}
}
deb/list_test.go
+8
View File
@@ -300,6 +300,14 @@ func (s *PackageListSuite) TestSearch(c *C) {
c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.2"}, true), Contains, []*Package{s.packages2[4], s.packages2[5]})
c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "1.1~bp1"}, true), Contains, []*Package{s.packages2[2], s.packages2[3], s.packages2[4], s.packages2[5]})
c.Check(s.il2.Search(Dependency{Architecture: "amd64", Pkg: "app", Relation: VersionGreaterOrEqual, Version: "5.0"}, true), IsNil)
// Provides with version number
python3CFFIBackend := &Package{Name: "python3-cffi-backend", Version: "1.15.1-5+b1", Architecture: "amd64", Provides: []string{"python3-cffi-backend-api-9729", "python3-cffi-backend-api-max (= 10495)", "python3-cffi-backend-api-min (= 9729)"}}
err := s.il2.Add(python3CFFIBackend)
c.Check(err, IsNil)
c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionGreaterOrEqual, Version: "9729"}, false), DeepEquals, []*Package{python3CFFIBackend})
c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionLess, Version: "9729"}, false), IsNil)
c.Check(s.il2.Search(Dependency{Pkg: "python3-cffi-backend-api-max", Relation: VersionDontCare}, false), DeepEquals, []*Package{python3CFFIBackend})
}
func (s *PackageListSuite) TestFilter(c *C) {
deb/local.go
+2 -2
View File
@@ -116,7 +116,7 @@ func (collection *LocalRepoCollection) search(filter func(*LocalRepo) bool, uniq
return result
}
collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
r := &LocalRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding local repo: %s\n", err)
@@ -219,7 +219,7 @@ func (collection *LocalRepoCollection) ByUUID(uuid string) (*LocalRepo, error) {
// ForEach runs method for each repository
func (collection *LocalRepoCollection) ForEach(handler func(*LocalRepo) error) error {
return collection.db.ProcessByPrefix([]byte("L"), func(key, blob []byte) error {
return collection.db.ProcessByPrefix([]byte("L"), func(_, blob []byte) error {
r := &LocalRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding repo: %s\n", err)
deb/package.go
+88 -14
View File
@@ -10,6 +10,7 @@ import (
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/utils"
"github.com/rs/zerolog/log"
)
// Package is single instance of Debian package
@@ -189,7 +190,12 @@ func NewInstallerPackageFromControlFile(input Stanza, repo *RemoteRepo, componen
return nil, err
}
relPath := filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
var relPath string
if repo.Distribution == aptly.DistributionFocal {
relPath = filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "legacy-images")
} else {
relPath = filepath.Join("dists", repo.Distribution, component, fmt.Sprintf("%s-%s", p.Name, architecture), "current", "images")
}
for i := range files {
files[i].downloadPath = relPath
@@ -307,6 +313,23 @@ func (p *Package) GetField(name string) string {
}
}
// ProvidedPackages returns just the package names of the provided packages (without version numbers such as
// `(= 1.2.3)`).
func (p *Package) ProvidedPackages() []string {
result := make([]string, len(p.Provides))
for i, provided := range p.Provides {
providedDep, err := ParseDependency(provided)
if err != nil {
// Should never happen, but I included this, so it definitely has the old behavior in case there is no
// special syntax.
result[i] = provided
} else {
result[i] = providedDep.Pkg
}
}
return result
}
// MatchesArchitecture checks whether packages matches specified architecture
func (p *Package) MatchesArchitecture(arch string) bool {
if p.Architecture == ArchitectureAll && arch != ArchitectureSource {
@@ -316,24 +339,75 @@ func (p *Package) MatchesArchitecture(arch string) bool {
return p.Architecture == arch
}
func JoinErrors(errs ...error) error {
var combinedErr error
for _, err := range errs {
if err != nil {
if combinedErr == nil {
combinedErr = err
} else {
combinedErr = fmt.Errorf("%w\n%v", combinedErr, err)
}
}
}
return combinedErr
}
// providesDependency checks if the package `Provide:`s the dependency, assuming that the architecture matches.
// If the `Provides:` entry includes a version number, it will be considered when checking the dependency.
func (p *Package) providesDependency(dep Dependency) (bool, error) {
var errs []error // won't cause an allocation in case of no errors
for _, provided := range p.Provides {
providedDep, err := ParseDependency(provided)
if err != nil {
errs = append(errs, err)
}
// The only relation allowed here is `=`.
// > The relations allowed are [...]. The exception is the Provides field, for which only = is allowed.
// > [...]
// > A Provides field may contain version numbers, and such a version number will be considered when
// > considering a dependency on or conflict with the virtual package name.
// -- https://www.debian.org/doc/debian-policy/ch-relationships.html
switch providedDep.Relation {
case VersionDontCare:
if providedDep.Pkg == dep.Pkg {
return true, nil
}
case VersionEqual:
providedVersion := providedDep.Version
if providedDep.Pkg == dep.Pkg && versionSatisfiesDependency(providedVersion, dep) {
return true, nil
}
default:
errs = append(errs, fmt.Errorf("unsupported relation in Provides: %s", providedDep.String()))
}
}
return false, JoinErrors(errs...)
}
// MatchesDependency checks whether package matches specified dependency
func (p *Package) MatchesDependency(dep Dependency) bool {
if dep.Architecture != "" && !p.MatchesArchitecture(dep.Architecture) {
return false
}
if dep.Relation == VersionDontCare {
if utils.StrSliceHasItem(p.Provides, dep.Pkg) {
return true
}
return dep.Pkg == p.Name
providesDep, err := p.providesDependency(dep)
if err != nil {
log.Warn().Err(err).Msg("Error while checking if package provides dependency")
}
if providesDep {
return true
}
if dep.Pkg != p.Name {
return false
}
r := CompareVersions(p.Version, dep.Version)
return versionSatisfiesDependency(p.Version, dep)
}
func versionSatisfiesDependency(version string, dep Dependency) bool {
r := CompareVersions(version, dep.Version)
switch dep.Relation {
case VersionEqual:
@@ -347,13 +421,15 @@ func (p *Package) MatchesDependency(dep Dependency) bool {
case VersionGreaterOrEqual:
return r >= 0
case VersionPatternMatch:
matched, err := filepath.Match(dep.Version, p.Version)
matched, err := filepath.Match(dep.Version, version)
return err == nil && matched
case VersionRegexp:
return dep.Regexp.FindStringIndex(p.Version) != nil
return dep.Regexp.FindStringIndex(version) != nil
case VersionDontCare:
return true
default:
panic(fmt.Sprintf("unknown relation: %d", dep.Relation))
}
panic("unknown relation")
}
// GetName returns package name
@@ -621,9 +697,7 @@ func (p *Package) LinkFromPool(publishedStorage aptly.PublishedStorage, packageP
return err
}
publishedDirectory := filepath.Join(prefix, relPath)
err = publishedStorage.LinkFromPool(publishedDirectory, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
err = publishedStorage.LinkFromPool(prefix, relPath, f.Filename, packagePool, sourcePoolPath, f.Checksums, force)
if err != nil {
return err
}
deb/package_collection.go
+1 -1
View File
@@ -317,7 +317,7 @@ func (collection *PackageCollection) Scan(q PackageQuery) (result *PackageList)
}
// Search is not implemented
func (collection *PackageCollection) Search(dep Dependency, allMatches bool) (searchResults []*Package) {
func (collection *PackageCollection) Search(_ Dependency, _ bool) (searchResults []*Package) {
panic("Not implemented")
}
deb/package_test.go
+4 -1
View File
@@ -333,9 +333,12 @@ func (s *PackageSuite) TestMatchesDependency(c *C) {
// Provides
c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, false)
p.Provides = []string{"fun", "game"}
p.Provides = []string{"fun (= 42)", "game"}
c.Check(p.MatchesDependency(Dependency{Pkg: "game", Relation: VersionDontCare}), Equals, true)
c.Check(p.MatchesDependency(Dependency{Pkg: "game", Architecture: "amd64", Relation: VersionDontCare}), Equals, false)
c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionDontCare}), Equals, true)
c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionGreaterOrEqual, Version: "42"}), Equals, true)
c.Check(p.MatchesDependency(Dependency{Pkg: "fun", Relation: VersionLess, Version: "42"}), Equals, false)
}
func (s *PackageSuite) TestGetDependencies(c *C) {
deb/publish.go
+133 -44
View File
@@ -5,7 +5,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
@@ -44,6 +43,7 @@ type PublishedRepo struct {
ButAutomaticUpgrades string
Label string
Suite string
Codename string
// Architectures is a list of all architectures published
Architectures []string
// SourceKind is "local"/"repo"
@@ -70,6 +70,9 @@ type PublishedRepo struct {
// Provide index files per hash also
AcquireByHash bool
// Support multiple distributions
MultiDist bool
}
// ParsePrefix splits [storage:]prefix into components
@@ -153,13 +156,14 @@ func walkUpTree(source interface{}, collectionFactory *CollectionFactory) (rootD
// distribution and architectures are user-defined properties
// components & sources are lists of component to source mapping (*Snapshot or *LocalRepo)
func NewPublishedRepo(storage, prefix, distribution string, architectures []string,
components []string, sources []interface{}, collectionFactory *CollectionFactory) (*PublishedRepo, error) {
components []string, sources []interface{}, collectionFactory *CollectionFactory, multiDist bool) (*PublishedRepo, error) {
result := &PublishedRepo{
UUID: uuid.New(),
Storage: storage,
Architectures: architectures,
Sources: make(map[string]string),
sourceItems: make(map[string]repoSourceItem),
MultiDist: multiDist,
}
if len(sources) == 0 {
@@ -197,9 +201,6 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
if distribution == "" || component == "" {
rootDistributions, rootComponents := walkUpTree(source, collectionFactory)
if distribution == "" {
for i := range rootDistributions {
rootDistributions[i] = strings.Replace(rootDistributions[i], "/", "-", -1)
}
discoveredDistributions = append(discoveredDistributions, rootDistributions...)
}
if component == "" {
@@ -264,10 +265,6 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
}
}
if strings.Contains(distribution, "/") {
return nil, fmt.Errorf("invalid distribution %s, '/' is not allowed", distribution)
}
result.Distribution = distribution
// only fields which are unique by all given snapshots are set on published
@@ -284,7 +281,7 @@ func NewPublishedRepo(storage, prefix, distribution string, architectures []stri
return result, nil
}
// MarshalJSON requires object to be "loaded completely"
// MarshalJSON requires object to filled by "LoadShallow" or "LoadComplete"
func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
type sourceInfo struct {
Component, Name string
@@ -312,6 +309,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
"Label": p.Label,
"Origin": p.Origin,
"Suite": p.Suite,
"Codename": p.Codename,
"NotAutomatic": p.NotAutomatic,
"ButAutomaticUpgrades": p.ButAutomaticUpgrades,
"Prefix": p.Prefix,
@@ -321,6 +319,7 @@ func (p *PublishedRepo) MarshalJSON() ([]byte, error) {
"Storage": p.Storage,
"SkipContents": p.SkipContents,
"AcquireByHash": p.AcquireByHash,
"MultiDist": p.MultiDist,
})
}
@@ -366,6 +365,10 @@ func (p *PublishedRepo) String() string {
extras = append(extras, fmt.Sprintf("suite: %s", p.Suite))
}
if p.Codename != "" {
extras = append(extras, fmt.Sprintf("codename: %s", p.Codename))
}
extra = strings.Join(extras, ", ")
if extra != "" {
@@ -418,6 +421,29 @@ func (p *PublishedRepo) Components() []string {
return result
}
// Components returns sorted list of published repo source names
func (p *PublishedRepo) SourceNames() []string {
var sources = []string{}
for _, component := range p.Components() {
var source string
item := p.sourceItems[component]
if item.snapshot != nil {
source = item.snapshot.Name
} else if item.localRepo != nil {
source = item.localRepo.Name
} else {
panic("no snapshot/localRepo")
}
sources = append(sources, fmt.Sprintf("%s:%s", source, component))
}
sort.Strings(sources)
return sources
}
// UpdateLocalRepo updates content from local repo in component
func (p *PublishedRepo) UpdateLocalRepo(component string) {
if p.SourceKind != SourceLocalRepo {
@@ -437,7 +463,10 @@ func (p *PublishedRepo) UpdateSnapshot(component string, snapshot *Snapshot) {
panic("not snapshot publish")
}
item := p.sourceItems[component]
item, exists := p.sourceItems[component]
if !exists {
item = repoSourceItem{}
}
item.snapshot = snapshot
p.sourceItems[component] = item
@@ -513,6 +542,14 @@ func (p *PublishedRepo) GetSuite() string {
return p.Suite
}
// GetCodename returns default or manual Codename:
func (p *PublishedRepo) GetCodename() string {
if p.Codename == "" {
return p.Distribution
}
return p.Codename
}
// Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them
func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider,
collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool) error {
@@ -582,7 +619,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
var tempDir string
tempDir, err = ioutil.TempDir(os.TempDir(), "aptly")
tempDir, err = os.MkdirTemp(os.TempDir(), "aptly")
if err != nil {
return err
}
@@ -605,7 +642,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
// For all architectures, pregenerate packages/sources files
for _, arch := range p.Architectures {
indexes.PackageIndex(component, arch, false, false)
indexes.PackageIndex(component, arch, false, false, p.Distribution)
}
list.PrepareIndex()
@@ -627,9 +664,18 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
if err2 != nil {
return err2
}
relPath = filepath.Join("pool", component, poolDir)
if p.MultiDist {
relPath = filepath.Join("pool", p.Distribution, component, poolDir)
} else {
relPath = filepath.Join("pool", component, poolDir)
}
} else {
relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
if p.Distribution == aptly.DistributionFocal {
relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "legacy-images")
} else {
relPath = filepath.Join("dists", p.Distribution, component, fmt.Sprintf("%s-%s", pkg.Name, arch), "current", "images")
}
}
err = pkg.LinkFromPool(publishedStorage, packagePool, p.Prefix, relPath, forceOverwrite)
@@ -667,7 +713,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
}
bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller).BufWriter()
bufWriter, err = indexes.PackageIndex(component, arch, pkg.IsUdeb, pkg.IsInstaller, p.Distribution).BufWriter()
if err != nil {
return err
}
@@ -721,7 +767,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
// For all architectures, pregenerate .udeb indexes
for _, arch := range p.Architectures {
indexes.PackageIndex(component, arch, true, false)
indexes.PackageIndex(component, arch, true, false, p.Distribution)
}
}
@@ -735,6 +781,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
release["Origin"] = p.GetOrigin()
release["Label"] = p.GetLabel()
release["Suite"] = p.GetSuite()
release["Codename"] = p.GetCodename()
if p.AcquireByHash {
release["Acquire-By-Hash"] = "yes"
}
@@ -793,7 +840,7 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP
}
release["Label"] = p.GetLabel()
release["Suite"] = p.GetSuite()
release["Codename"] = p.Distribution
release["Codename"] = p.GetCodename()
release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST")
release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ")
if p.AcquireByHash {
@@ -953,8 +1000,11 @@ func (collection *PublishedRepoCollection) Update(repo *PublishedRepo) error {
return batch.Write()
}
// LoadComplete loads additional information for remote repo
func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
// LoadShallow loads basic information on the repo's sources
//
// This does not *fully* load in the sources themselves and their packages.
// It's useful if you just want to use JSON serialization without loading in unnecessary things.
func (collection *PublishedRepoCollection) LoadShallow(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
repo.sourceItems = make(map[string]repoSourceItem)
if repo.SourceKind == SourceSnapshot {
@@ -965,10 +1015,6 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
if err != nil {
return
}
err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot)
if err != nil {
return
}
repo.sourceItems[component] = item
}
@@ -980,6 +1026,30 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
if err != nil {
return
}
item.packageRefs = &PackageRefList{}
repo.sourceItems[component] = item
}
} else {
panic("unknown SourceKind")
}
return
}
// LoadComplete loads complete information on the sources of the repo *and* their packages
func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, collectionFactory *CollectionFactory) (err error) {
collection.LoadShallow(repo, collectionFactory)
if repo.SourceKind == SourceSnapshot {
for _, item := range repo.sourceItems {
err = collectionFactory.SnapshotCollection().LoadComplete(item.snapshot)
if err != nil {
return
}
}
} else if repo.SourceKind == SourceLocalRepo {
for component, item := range repo.sourceItems {
err = collectionFactory.LocalRepoCollection().LoadComplete(item.localRepo)
if err != nil {
return
@@ -998,13 +1068,10 @@ func (collection *PublishedRepoCollection) LoadComplete(repo *PublishedRepo, col
}
}
item.packageRefs = &PackageRefList{}
err = item.packageRefs.Decode(encoded)
if err != nil {
return
}
repo.sourceItems[component] = item
}
} else {
panic("unknown SourceKind")
@@ -1086,7 +1153,7 @@ func (collection *PublishedRepoCollection) ByLocalRepo(repo *LocalRepo) []*Publi
// ForEach runs method for each repository
func (collection *PublishedRepoCollection) ForEach(handler func(*PublishedRepo) error) error {
return collection.db.ProcessByPrefix([]byte("U"), func(key, blob []byte) error {
return collection.db.ProcessByPrefix([]byte("U"), func(_, blob []byte) error {
r := &PublishedRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding published repo: %s\n", err)
@@ -1104,18 +1171,10 @@ func (collection *PublishedRepoCollection) Len() int {
return len(collection.list)
}
// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
collection.loadList()
var err error
func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix string, components []string,
collectionFactory *CollectionFactory, progress aptly.Progress) (map[string][]string, error) {
referencedFiles := map[string][]string{}
if progress != nil {
progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", "))
}
processedComponentRefs := map[string]*PackageRefList{}
for _, r := range collection.list {
if r.Prefix == prefix {
@@ -1134,16 +1193,28 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
continue
}
err = collection.LoadComplete(r, collectionFactory)
if err != nil {
return err
if err := collection.LoadComplete(r, collectionFactory); err != nil {
return nil, err
}
for _, component := range components {
if utils.StrSliceHasItem(repoComponents, component) {
packageList, err := NewPackageListFromRefList(r.RefList(component), collectionFactory.PackageCollection(), progress)
unseenRefs := r.RefList(component)
processedRefs := processedComponentRefs[component]
if processedRefs != nil {
unseenRefs = unseenRefs.Subtract(processedRefs)
} else {
processedRefs = NewPackageRefList()
}
if unseenRefs.Len() == 0 {
continue
}
processedComponentRefs[component] = processedRefs.Merge(unseenRefs, false, true)
packageList, err := NewPackageListFromRefList(unseenRefs, collectionFactory.PackageCollection(), progress)
if err != nil {
return err
return nil, err
}
packageList.ForEach(func(p *Package) error {
@@ -1163,6 +1234,24 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st
}
}
return referencedFiles, nil
}
// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair
func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string,
publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error {
collection.loadList()
if progress != nil {
progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", "))
}
referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress)
if err != nil {
return err
}
for _, component := range components {
sort.Strings(referencedFiles[component])
deb/publish_bench_test.go
+113
View File
@@ -0,0 +1,113 @@
package deb
import (
"fmt"
"os"
"sort"
"testing"
"github.com/aptly-dev/aptly/database/goleveldb"
)
func BenchmarkListReferencedFiles(b *testing.B) {
const defaultComponent = "main"
const repoCount = 16
const repoPackagesCount = 1024
const uniqPackagesCount = 64
tmpDir, err := os.MkdirTemp("", "aptly-bench")
if err != nil {
b.Fatal(err)
}
defer os.RemoveAll(tmpDir)
db, err := goleveldb.NewOpenDB(tmpDir)
if err != nil {
b.Fatal(err)
}
defer db.Close()
factory := NewCollectionFactory(db)
packageCollection := factory.PackageCollection()
repoCollection := factory.LocalRepoCollection()
publishCollection := factory.PublishedRepoCollection()
sharedRefs := NewPackageRefList()
{
transaction, err := db.OpenTransaction()
if err != nil {
b.Fatal(err)
}
for pkgIndex := 0; pkgIndex < repoPackagesCount-uniqPackagesCount; pkgIndex++ {
p := &Package{
Name: fmt.Sprintf("pkg-shared_%d", pkgIndex),
Version: "1",
Architecture: "amd64",
}
p.UpdateFiles(PackageFiles{PackageFile{
Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex),
}})
packageCollection.UpdateInTransaction(p, transaction)
sharedRefs.Refs = append(sharedRefs.Refs, p.Key(""))
}
sort.Sort(sharedRefs)
if err := transaction.Commit(); err != nil {
b.Fatal(err)
}
}
for repoIndex := 0; repoIndex < repoCount; repoIndex++ {
refs := NewPackageRefList()
transaction, err := db.OpenTransaction()
if err != nil {
b.Fatal(err)
}
for pkgIndex := 0; pkgIndex < uniqPackagesCount; pkgIndex++ {
p := &Package{
Name: fmt.Sprintf("pkg%d_%d", repoIndex, pkgIndex),
Version: "1",
Architecture: "amd64",
}
p.UpdateFiles(PackageFiles{PackageFile{
Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex),
}})
packageCollection.UpdateInTransaction(p, transaction)
refs.Refs = append(refs.Refs, p.Key(""))
}
if err := transaction.Commit(); err != nil {
b.Fatal(err)
}
sort.Sort(refs)
repo := NewLocalRepo(fmt.Sprintf("repo%d", repoIndex), "comment")
repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex)
repo.DefaultComponent = defaultComponent
repo.UpdateRefList(refs.Merge(sharedRefs, false, true))
repoCollection.Add(repo)
publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory, false)
if err != nil {
b.Fatal(err)
}
publishCollection.Add(publish)
}
db.CompactDB()
b.ResetTimer()
for i := 0; i < b.N; i++ {
_, err := publishCollection.listReferencedFilesByComponent("test", []string{defaultComponent}, factory, nil)
if err != nil {
b.Fatal(err)
}
}
}
deb/publish_test.go
+144 -37
View File
@@ -7,6 +7,7 @@ import (
"io/ioutil"
"os"
"path/filepath"
"sort"
"github.com/aptly-dev/aptly/aptly"
"github.com/aptly-dev/aptly/database"
@@ -133,19 +134,19 @@ func (s *PublishedRepoSuite) SetUpTest(c *C) {
s.packageCollection.Update(s.p2)
s.packageCollection.Update(s.p3)
s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
s.repo, _ = NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
s.repo.SkipContents = true
s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo2, _ = NewPublishedRepo("", "ppa", "maverick", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
s.repo2.SkipContents = true
s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
s.repo3, _ = NewPublishedRepo("", "linux", "natty", nil, []string{"main", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
s.repo3.SkipContents = true
s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo4, _ = NewPublishedRepo("", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
s.repo4.SkipContents = true
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "maverick", []string{"source"}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
s.repo5.SkipContents = true
}
@@ -177,19 +178,71 @@ func (s *PublishedRepoSuite) TestNewPublishedRepo(c *C) {
c.Check(s.repo3.RefList("main").Len(), Equals, 3)
c.Check(s.repo3.RefList("contrib").Len(), Equals, 3)
c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory) }, PanicMatches, "publish with empty sources")
c.Check(func() { NewPublishedRepo("", ".", "a", nil, nil, nil, s.factory, false) }, PanicMatches, "publish with empty sources")
c.Check(func() {
NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
NewPublishedRepo("", ".", "a", nil, []string{"main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
}, PanicMatches, "sources and components should be equal in size")
c.Check(func() {
NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory)
NewPublishedRepo("", ".", "a", nil, []string{"main", "contrib"}, []interface{}{s.localRepo, s.snapshot2}, s.factory, false)
}, PanicMatches, "interface conversion:.*")
_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
_, err := NewPublishedRepo("", ".", "a", nil, []string{"main", "main"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
c.Check(err, ErrorMatches, "duplicate component name: main")
_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
c.Check(err, ErrorMatches, "invalid distribution wheezy/updates, '/' is not allowed")
_, err = NewPublishedRepo("", ".", "wheezy/updates", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
c.Check(err, IsNil)
}
func (s *PublishedRepoSuite) TestMultiDistPool(c *C) {
repo, err := NewPublishedRepo("", "ppa", "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, true)
c.Assert(err, IsNil)
err = repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false)
c.Assert(err, IsNil)
publishedStorage := files.NewPublishedStorage(s.root, "", "")
c.Check(repo.Architectures, DeepEquals, []string{"i386"})
rf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/Release"))
c.Assert(err, IsNil)
cfr := NewControlFileReader(rf, true, false)
st, err := cfr.ReadStanza()
c.Assert(err, IsNil)
c.Check(st["Origin"], Equals, "ppa squeeze")
c.Check(st["Components"], Equals, "main")
c.Check(st["Architectures"], Equals, "i386")
pf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Packages"))
c.Assert(err, IsNil)
cfr = NewControlFileReader(pf, false, false)
for i := 0; i < 3; i++ {
st, err = cfr.ReadStanza()
c.Assert(err, IsNil)
c.Check(st["Filename"], Equals, "pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb")
}
st, err = cfr.ReadStanza()
c.Assert(err, IsNil)
c.Assert(st, IsNil)
drf, err := os.Open(filepath.Join(publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"))
c.Assert(err, IsNil)
cfr = NewControlFileReader(drf, true, false)
st, err = cfr.ReadStanza()
c.Assert(err, IsNil)
c.Check(st["Archive"], Equals, "squeeze")
c.Check(st["Architecture"], Equals, "i386")
_, err = os.Stat(filepath.Join(publishedStorage.PublicPath(), "ppa/pool/squeeze/main/a/alien-arena/alien-arena-common_7.40-2_i386.deb"))
c.Assert(err, IsNil)
}
func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
@@ -248,7 +301,7 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
errorExpected: "invalid prefix .*",
},
} {
repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory)
repo, err := NewPublishedRepo("", t.prefix, "squeeze", nil, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
if t.errorExpected != "" {
c.Check(err, ErrorMatches, t.errorExpected)
} else {
@@ -258,51 +311,51 @@ func (s *PublishedRepoSuite) TestPrefixNormalization(c *C) {
}
func (s *PublishedRepoSuite) TestDistributionComponentGuessing(c *C) {
repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
repo, err := NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.snapshot}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Components(), DeepEquals, []string{"main"})
repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "wheezy", nil, []string{""}, []interface{}{s.snapshot}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "wheezy")
c.Check(repo.Components(), DeepEquals, []string{"main"})
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"non-free"}, []interface{}{s.snapshot}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Components(), DeepEquals, []string{"non-free"})
repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "squeeze", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Components(), DeepEquals, []string{"main"})
_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory)
_, err = NewPublishedRepo("", "ppa", "", nil, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
c.Check(err, ErrorMatches, "unable to guess distribution name, please specify explicitly")
s.localRepo.DefaultDistribution = "precise"
s.localRepo.DefaultComponent = "contrib"
s.factory.LocalRepoCollection().Update(s.localRepo)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "precise")
c.Check(repo.Components(), DeepEquals, []string{"contrib"})
s.localRepo.DefaultDistribution = "precise/updates"
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{""}, []interface{}{s.localRepo}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "precise-updates")
c.Check(repo.Distribution, Equals, "precise/updates")
c.Check(repo.Components(), DeepEquals, []string{"contrib"})
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
repo, err = NewPublishedRepo("", "ppa", "", nil, []string{"", "contrib"}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
c.Check(err, IsNil)
c.Check(repo.Distribution, Equals, "squeeze")
c.Check(repo.Components(), DeepEquals, []string{"contrib", "main"})
_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory)
_, err = NewPublishedRepo("", "ppa", "", nil, []string{"", ""}, []interface{}{s.snapshot, s.snapshot2}, s.factory, false)
c.Check(err, ErrorMatches, "duplicate component name: main")
}
@@ -390,10 +443,10 @@ func (s *PublishedRepoSuite) TestString(c *C) {
"ppa/squeeze [] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
c.Check(s.repo2.String(), Equals,
"ppa/maverick [] publishes {main: [local1]: comment1}")
repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
repo, _ := NewPublishedRepo("", "", "squeeze", []string{"s390"}, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
c.Check(repo.String(), Equals,
"./squeeze [s390] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory)
repo, _ = NewPublishedRepo("", "", "squeeze", []string{"i386", "amd64"}, []string{"main"}, []interface{}{s.snapshot}, s.factory, false)
c.Check(repo.String(), Equals,
"./squeeze [i386, amd64] publishes {main: [snap]: Snapshot from mirror [yandex]: http://mirror.yandex.ru/debian/ squeeze}")
repo.Origin = "myorigin"
@@ -450,13 +503,22 @@ type PublishedRepoCollectionSuite struct {
var _ = Suite(&PublishedRepoCollectionSuite{})
func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.SetUpPackages()
s.db, _ = goleveldb.NewOpenDB(c.MkDir())
s.factory = NewCollectionFactory(s.db)
s.snapshotCollection = s.factory.SnapshotCollection()
s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1")
s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2")
snap1Refs := NewPackageRefList()
snap1Refs.Refs = [][]byte{s.p1.Key(""), s.p2.Key("")}
sort.Sort(snap1Refs)
s.snap1 = NewSnapshotFromRefList("snap1", []*Snapshot{}, snap1Refs, "desc1")
snap2Refs := NewPackageRefList()
snap2Refs.Refs = [][]byte{s.p3.Key("")}
sort.Sort(snap2Refs)
s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2")
s.snapshotCollection.Add(s.snap1)
s.snapshotCollection.Add(s.snap2)
@@ -464,11 +526,11 @@ func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) {
s.localRepo = NewLocalRepo("local1", "comment1")
s.factory.LocalRepoCollection().Add(s.localRepo)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory)
s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory)
s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main", "contrib"}, []interface{}{s.snap2, s.snap1}, s.factory, false)
s.repo3, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap2}, s.factory, false)
s.repo4, _ = NewPublishedRepo("", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "precise", []string{}, []string{"main"}, []interface{}{s.localRepo}, s.factory, false)
s.collection = s.factory.PublishedRepoCollection()
}
@@ -534,7 +596,7 @@ func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) {
c.Assert(r.sourceItems["main"].snapshot, IsNil)
c.Assert(s.collection.LoadComplete(r, s.factory), IsNil)
c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID)
c.Assert(r.RefList("main").Len(), Equals, 0)
c.Assert(r.RefList("main").Len(), Equals, 2)
r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise")
c.Assert(err, IsNil)
@@ -625,6 +687,51 @@ func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) {
c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5})
}
func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) {
c.Check(s.factory.PackageCollection().Update(s.p1), IsNil)
c.Check(s.factory.PackageCollection().Update(s.p2), IsNil)
c.Check(s.factory.PackageCollection().Update(s.p3), IsNil)
c.Check(s.collection.Add(s.repo1), IsNil)
c.Check(s.collection.Add(s.repo2), IsNil)
c.Check(s.collection.Add(s.repo4), IsNil)
c.Check(s.collection.Add(s.repo5), IsNil)
files, err := s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
c.Assert(err, IsNil)
for _, v := range files {
sort.Strings(v)
}
c.Check(files, DeepEquals, map[string][]string{
"contrib": {
"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
"a/alien-arena/mars-invaders_7.40-2_i386.deb",
},
"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
})
snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3")
s.snapshotCollection.Add(snap3)
// Ensure that adding a second publish point with matching files doesn't give duplicate results.
repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false)
c.Check(err, IsNil)
c.Check(s.collection.Add(repo3), IsNil)
files, err = s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil)
c.Assert(err, IsNil)
for _, v := range files {
sort.Strings(v)
}
c.Check(files, DeepEquals, map[string][]string{
"contrib": {
"a/alien-arena/alien-arena-common_7.40-2_i386.deb",
"a/alien-arena/mars-invaders_7.40-2_i386.deb",
},
"main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"},
})
}
type PublishedRepoRemoveSuite struct {
PackageListMixinSuite
db database.Storage
@@ -650,11 +757,11 @@ func (s *PublishedRepoRemoveSuite) SetUpTest(c *C) {
s.snapshotCollection.Add(s.snap1)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory)
s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory)
s.repo1, _ = NewPublishedRepo("", "ppa", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo2, _ = NewPublishedRepo("", "", "anaconda", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo3, _ = NewPublishedRepo("", "ppa", "meduza", []string{}, []string{"main"}, []interface{}{s.snap1}, s.factory, false)
s.repo4, _ = NewPublishedRepo("", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
s.repo5, _ = NewPublishedRepo("files:other", "ppa", "osminog", []string{}, []string{"contrib"}, []interface{}{s.snap1}, s.factory, false)
s.collection = s.factory.PublishedRepoCollection()
s.collection.Add(s.repo1)
deb/query.go
+5 -5
View File
@@ -138,7 +138,7 @@ func (q *NotQuery) Matches(pkg PackageLike) bool {
}
// Fast is false
func (q *NotQuery) Fast(list PackageCatalog) bool {
func (q *NotQuery) Fast(_ PackageCatalog) bool {
return false
}
@@ -197,7 +197,7 @@ func (q *FieldQuery) Query(list PackageCatalog) (result *PackageList) {
}
// Fast depends on the query
func (q *FieldQuery) Fast(list PackageCatalog) bool {
func (q *FieldQuery) Fast(_ PackageCatalog) bool {
return false
}
@@ -265,7 +265,7 @@ func (q *PkgQuery) Matches(pkg PackageLike) bool {
}
// Fast is always true for package query
func (q *PkgQuery) Fast(list PackageCatalog) bool {
func (q *PkgQuery) Fast(_ PackageCatalog) bool {
return true
}
@@ -280,12 +280,12 @@ func (q *PkgQuery) String() string {
}
// Matches on specific properties
func (q *MatchAllQuery) Matches(pkg PackageLike) bool {
func (q *MatchAllQuery) Matches(_ PackageLike) bool {
return true
}
// Fast is always true for match all query
func (q *MatchAllQuery) Fast(list PackageCatalog) bool {
func (q *MatchAllQuery) Fast(_ PackageCatalog) bool {
return true
}
deb/reflist.go
+43 -48
View File
@@ -71,7 +71,9 @@ func (l *PackageRefList) Encode() []byte {
// Decode decodes msgpack representation into PackageRefLit
func (l *PackageRefList) Decode(input []byte) error {
decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{})
handle := &codec.MsgpackHandle{}
handle.ZeroCopy = true
decoder := codec.NewDecoderBytes(input, handle)
return decoder.Decode(l)
}
@@ -194,31 +196,21 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
// until we reached end of both lists
for il < ll || ir < lr {
// if we've exhausted left list, pull the rest from the right
if il == ll {
pr, err = packageCollection.ByKey(r.Refs[ir])
if err != nil {
return nil, err
}
result = append(result, PackageDiff{Left: nil, Right: pr})
ir++
continue
var rl, rr []byte
if il < ll {
rl = l.Refs[il]
}
// if we've exhausted right list, pull the rest from the left
if ir == lr {
pl, err = packageCollection.ByKey(l.Refs[il])
if err != nil {
return nil, err
}
result = append(result, PackageDiff{Left: pl, Right: nil})
il++
continue
if ir < lr {
rr = r.Refs[ir]
}
// refs on both sides are present, load them
rl, rr := l.Refs[il], r.Refs[ir]
// compare refs
rel := bytes.Compare(rl, rr)
// an unset ref is less than all others, but since it represents the end
// of a reflist, it should be *greater*, so flip the comparison result
if rl == nil || rr == nil {
rel = -rel
}
if rel == 0 {
// refs are identical, so are packages, advance pointer
@@ -227,14 +219,14 @@ func (l *PackageRefList) Diff(r *PackageRefList, packageCollection *PackageColle
pl, pr = nil, nil
} else {
// load pl & pr if they haven't been loaded before
if pl == nil {
if pl == nil && rl != nil {
pl, err = packageCollection.ByKey(rl)
if err != nil {
return nil, err
}
}
if pr == nil {
if pr == nil && rr != nil {
pr, err = packageCollection.ByKey(rr)
if err != nil {
return nil, err
@@ -310,38 +302,41 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConfli
overridenName = nil
overriddenArch = nil
} else {
partsL := bytes.Split(rl, []byte(" "))
archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
if !ignoreConflicting || overrideMatching {
partsL := bytes.Split(rl, []byte(" "))
archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2]
partsR := bytes.Split(rr, []byte(" "))
archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
partsR := bytes.Split(rr, []byte(" "))
archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2]
if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
// conflicting duplicates with same arch, name, version, but different file hash
result.Refs = append(result.Refs, r.Refs[ir])
il++
ir++
overridenName = nil
overriddenArch = nil
continue
}
if overrideMatching {
if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
// this package has already been overridden on the right
il++
continue
}
if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
// override with package from the right
if !ignoreConflicting && bytes.Equal(archL, archR) &&
bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) {
// conflicting duplicates with same arch, name, version, but different file hash
result.Refs = append(result.Refs, r.Refs[ir])
il++
ir++
overriddenArch = archL
overridenName = nameL
overridenName = nil
overriddenArch = nil
continue
}
if overrideMatching {
if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) {
// this package has already been overridden on the right
il++
continue
}
if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) {
// override with package from the right
result.Refs = append(result.Refs, r.Refs[ir])
il++
ir++
overriddenArch = archL
overridenName = nameL
continue
}
}
}
// otherwise append smallest of two
deb/reflist_bench_test.go
+47
View File
@@ -0,0 +1,47 @@
package deb
import (
"fmt"
"sort"
"testing"
)
func BenchmarkReflistSimpleMerge(b *testing.B) {
const count = 4096
l := NewPackageRefList()
r := NewPackageRefList()
for i := 0; i < count; i++ {
if i%2 == 0 {
l.Refs = append(l.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
} else {
r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
}
}
sort.Sort(l)
sort.Sort(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
l.Merge(r, false, true)
}
}
func BenchmarkReflistDecode(b *testing.B) {
const count = 4096
r := NewPackageRefList()
for i := 0; i < count; i++ {
r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i)))
}
sort.Sort(r)
data := r.Encode()
b.ResetTimer()
for i := 0; i < b.N; i++ {
(&PackageRefList{}).Decode(data)
}
}
deb/reflist_test.go
+35
View File
@@ -237,6 +237,41 @@ func (s *PackageRefListSuite) TestDiff(c *C) {
}
func (s *PackageRefListSuite) TestDiffCompactsAtEnd(c *C) {
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
packages := []*Package{
{Name: "app", Version: "1.1~bp1", Architecture: "i386"}, //0
{Name: "app", Version: "1.1~bp2", Architecture: "i386"}, //1
{Name: "app", Version: "1.1~bp2", Architecture: "amd64"}, //2
}
for _, p := range packages {
coll.Update(p)
}
listA := NewPackageList()
listA.Add(packages[0])
listB := NewPackageList()
listB.Add(packages[1])
listB.Add(packages[2])
reflistA := NewPackageRefListFromPackageList(listA)
reflistB := NewPackageRefListFromPackageList(listB)
diffAB, err := reflistA.Diff(reflistB, coll)
c.Check(err, IsNil)
c.Check(diffAB, HasLen, 2)
c.Check(diffAB[0].Left, IsNil)
c.Check(diffAB[0].Right.String(), Equals, "app_1.1~bp2_amd64")
c.Check(diffAB[1].Left.String(), Equals, "app_1.1~bp1_i386")
c.Check(diffAB[1].Right.String(), Equals, "app_1.1~bp2_i386")
}
func (s *PackageRefListSuite) TestMerge(c *C) {
db, _ := goleveldb.NewOpenDB(c.MkDir())
coll := NewPackageCollection(db)
deb/remote.go
+24 -10
View File
@@ -259,6 +259,9 @@ func (repo *RemoteRepo) UdebPath(component string, architecture string) string {
// InstallerPath returns path of Packages files for given component and
// architecture
func (repo *RemoteRepo) InstallerPath(component string, architecture string) string {
if repo.Distribution == aptly.DistributionFocal {
return fmt.Sprintf("%s/installer-%s/current/legacy-images/SHA256SUMS", component, architecture)
}
return fmt.Sprintf("%s/installer-%s/current/images/SHA256SUMS", component, architecture)
}
@@ -270,17 +273,29 @@ func (repo *RemoteRepo) PackageURL(filename string) *url.URL {
}
// Fetch updates information about repository
func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier) error {
func (repo *RemoteRepo) Fetch(d aptly.Downloader, verifier pgp.Verifier, ignoreSignatures bool) error {
var (
release, inrelease, releasesig *os.File
err error
)
if verifier == nil {
if ignoreSignatures {
// 0. Just download release file to temporary URL
release, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("Release").String())
if err != nil {
return err
// 0.1 try downloading InRelease, ignore and strip signature
inrelease, err = http.DownloadTemp(gocontext.TODO(), d, repo.ReleaseURL("InRelease").String())
if err != nil {
return err
}
if verifier == nil {
return fmt.Errorf("no verifier specified")
}
release, err = verifier.ExtractClearsigned(inrelease)
if err != nil {
return err
}
goto ok
}
} else {
// 1. try InRelease file
@@ -428,8 +443,7 @@ ok:
}
// DownloadPackageIndexes downloads & parses package index files
func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, collectionFactory *CollectionFactory,
ignoreMismatch bool) error {
func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.Downloader, verifier pgp.Verifier, _ *CollectionFactory, ignoreSignatures bool, ignoreChecksums bool) error {
if repo.packageList != nil {
panic("packageList != nil")
}
@@ -462,14 +476,14 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
for _, info := range packagesPaths {
path, kind, component, architecture := info[0], info[1], info[2], info[3]
packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreMismatch)
packagesReader, packagesFile, err := http.DownloadTryCompression(gocontext.TODO(), d, repo.IndexesRootURL(), path, repo.ReleaseFiles, ignoreChecksums)
isInstaller := kind == PackageTypeInstaller
if err != nil {
if _, ok := err.(*http.NoCandidateFoundError); isInstaller && ok {
// checking if gpg file is only needed when checksums matches are required.
// otherwise there actually has been no candidate found and we can continue
if ignoreMismatch {
if ignoreChecksums {
continue
}
@@ -486,7 +500,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly.
return err
}
if verifier != nil {
if verifier != nil && !ignoreSignatures {
hashsumGpgPath := repo.IndexesRootURL().ResolveReference(&url.URL{Path: path + ".gpg"}).String()
var filesig *os.File
filesig, err = http.DownloadTemp(gocontext.TODO(), d, hashsumGpgPath)
@@ -777,7 +791,7 @@ func (collection *RemoteRepoCollection) search(filter func(*RemoteRepo) bool, un
return result
}
collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
collection.db.ProcessByPrefix([]byte("R"), func(_, blob []byte) error {
r := &RemoteRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding remote repo: %s\n", err)
@@ -880,7 +894,7 @@ func (collection *RemoteRepoCollection) ByUUID(uuid string) (*RemoteRepo, error)
// ForEach runs method for each repository
func (collection *RemoteRepoCollection) ForEach(handler func(*RemoteRepo) error) error {
return collection.db.ProcessByPrefix([]byte("R"), func(key, blob []byte) error {
return collection.db.ProcessByPrefix([]byte("R"), func(_, blob []byte) error {
r := &RemoteRepo{}
if err := r.Decode(blob); err != nil {
log.Printf("Error decoding mirror: %s\n", err)

Some files were not shown because too many files have changed in this diff Show More